diff --git a/404.html b/404.html
index fae2a54..ee5371f 100644
--- a/404.html
+++ b/404.html
@@ -360,6 +360,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="/tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4447,6 +4464,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="/tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4464,7 +4501,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4480,14 +4517,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4553,7 +4590,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4569,14 +4606,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4632,7 +4669,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4648,14 +4685,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/contributing/contributing/index.html b/apiv2/contributing/contributing/index.html
index 39511f0..d1505be 100644
--- a/apiv2/contributing/contributing/index.html
+++ b/apiv2/contributing/contributing/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" checked>
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/contributing/documentation/index.html b/apiv2/contributing/documentation/index.html
index b6b5501..90bab15 100644
--- a/apiv2/contributing/documentation/index.html
+++ b/apiv2/contributing/documentation/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" checked>
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/contributing/index.html b/apiv2/contributing/index.html
index 478c847..e414a6a 100644
--- a/apiv2/contributing/index.html
+++ b/apiv2/contributing/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" checked>
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link md-nav__link--active" for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link md-nav__link--active" for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/contributing/project_overview/index.html b/apiv2/contributing/project_overview/index.html
index 9c2bb9d..591c0a2 100644
--- a/apiv2/contributing/project_overview/index.html
+++ b/apiv2/contributing/project_overview/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" checked>
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/contributing/tests/index.html b/apiv2/contributing/tests/index.html
index bfb0755..118c19a 100644
--- a/apiv2/contributing/tests/index.html
+++ b/apiv2/contributing/tests/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" checked>
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/index.html b/apiv2/index.html
index 4b760af..5d20490 100644
--- a/apiv2/index.html
+++ b/apiv2/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link md-nav__link--active" for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link md-nav__link--active" for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4650,7 +4687,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4666,14 +4703,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/installation/index.html b/apiv2/installation/index.html
index 01ae151..6fba58a 100644
--- a/apiv2/installation/index.html
+++ b/apiv2/installation/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4741,7 +4778,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4757,14 +4794,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/apiv2/migration/index.html b/apiv2/migration/index.html
index 09de1a9..12f685c 100644
--- a/apiv2/migration/index.html
+++ b/apiv2/migration/index.html
@@ -369,6 +369,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4458,6 +4475,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4475,7 +4512,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4491,14 +4528,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4569,7 +4606,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" checked>
         
           
           
@@ -4585,14 +4622,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4648,7 +4685,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4664,14 +4701,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/benchmark/automl/AutoML-Benchmark/index.html b/benchmark/automl/AutoML-Benchmark/index.html
index 4ed0f42..ed0027d 100644
--- a/benchmark/automl/AutoML-Benchmark/index.html
+++ b/benchmark/automl/AutoML-Benchmark/index.html
@@ -375,6 +375,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4560,6 +4577,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4577,7 +4614,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4593,14 +4630,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4666,7 +4703,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4682,14 +4719,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4745,7 +4782,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4761,14 +4798,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/benchmark/automl/basic_example/index.html b/benchmark/automl/basic_example/index.html
index ab9f140..a8a0831 100644
--- a/benchmark/automl/basic_example/index.html
+++ b/benchmark/automl/basic_example/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4611,6 +4628,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4628,7 +4665,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4644,14 +4681,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4717,7 +4754,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4733,14 +4770,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4796,7 +4833,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4812,14 +4849,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/benchmark/automl/benchmark_on_openml/index.html b/benchmark/automl/benchmark_on_openml/index.html
index dc3f882..17263f0 100644
--- a/benchmark/automl/benchmark_on_openml/index.html
+++ b/benchmark/automl/benchmark_on_openml/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4477,6 +4494,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4494,7 +4531,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4510,14 +4547,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4583,7 +4620,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4599,14 +4636,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4662,7 +4699,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4678,14 +4715,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/benchmark/automl/important_params/index.html b/benchmark/automl/important_params/index.html
index 9ac320e..b7aba98 100644
--- a/benchmark/automl/important_params/index.html
+++ b/benchmark/automl/important_params/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4477,6 +4494,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4494,7 +4531,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4510,14 +4547,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4583,7 +4620,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4599,14 +4636,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4662,7 +4699,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4678,14 +4715,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/benchmark/automl/specific_task_fold_example/index.html b/benchmark/automl/specific_task_fold_example/index.html
index 59d2113..c7e97d5 100644
--- a/benchmark/automl/specific_task_fold_example/index.html
+++ b/benchmark/automl/specific_task_fold_example/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/benchmark/index.html b/benchmark/index.html
index 138cb20..898abf2 100644
--- a/benchmark/index.html
+++ b/benchmark/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4465,6 +4482,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4482,7 +4519,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4498,14 +4535,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4650,7 +4687,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4666,14 +4703,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5660,6 +5697,29 @@ <h2 id="need-help">Need help?<a class="headerlink" href="#need-help" title="Perm
     
     
     
+      
+  <span class="md-source-file__fact">
+    
+      
+  <span class="md-icon" title="Contributors">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2"/></svg>
+  </span>
+  <span>GitHub</span>
+
+    
+    <nav>
+      
+        <a href="https://github.com/joaquinvanschoren" class="md-author" title="@joaquinvanschoren">
+          
+          <img src="https://avatars.githubusercontent.com/u/1724503?v=4&size=72" alt="joaquinvanschoren">
+        </a>
+      
+      
+      
+    </nav>
+  </span>
+
+    
   </aside>
 
 
diff --git a/concepts/authentication/index.html b/concepts/authentication/index.html
index aa3081c..40a7558 100644
--- a/concepts/authentication/index.html
+++ b/concepts/authentication/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4516,6 +4533,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4533,7 +4570,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4549,14 +4586,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4622,7 +4659,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4638,14 +4675,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4701,7 +4738,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4717,14 +4754,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/benchmarking/index.html b/concepts/benchmarking/index.html
index 6c95164..1ee4474 100644
--- a/concepts/benchmarking/index.html
+++ b/concepts/benchmarking/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4525,6 +4542,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4542,7 +4579,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4558,14 +4595,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4631,7 +4668,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4647,14 +4684,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4710,7 +4747,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4726,14 +4763,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/data/index.html b/concepts/data/index.html
index 82b637b..f114d60 100644
--- a/concepts/data/index.html
+++ b/concepts/data/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4561,6 +4578,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4578,7 +4615,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4594,14 +4631,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4667,7 +4704,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4683,14 +4720,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4746,7 +4783,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4762,14 +4799,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/flows/index.html b/concepts/flows/index.html
index 8573cb5..affc645 100644
--- a/concepts/flows/index.html
+++ b/concepts/flows/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4534,6 +4551,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4551,7 +4588,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4567,14 +4604,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4640,7 +4677,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4656,14 +4693,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4719,7 +4756,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4735,14 +4772,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/index.html b/concepts/index.html
index 08732c6..4480262 100644
--- a/concepts/index.html
+++ b/concepts/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4467,6 +4484,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4484,7 +4521,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4500,14 +4537,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4573,7 +4610,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4589,14 +4626,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/runs/index.html b/concepts/runs/index.html
index 6d784c8..ce69b54 100644
--- a/concepts/runs/index.html
+++ b/concepts/runs/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4534,6 +4551,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4551,7 +4588,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4567,14 +4604,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4640,7 +4677,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4656,14 +4693,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4719,7 +4756,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4735,14 +4772,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/sharing/index.html b/concepts/sharing/index.html
index df47f95..3e63b1b 100644
--- a/concepts/sharing/index.html
+++ b/concepts/sharing/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4477,6 +4494,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4494,7 +4531,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4510,14 +4547,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4583,7 +4620,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4599,14 +4636,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4662,7 +4699,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4678,14 +4715,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/tagging/index.html b/concepts/tagging/index.html
index 0d96110..9cf0ce0 100644
--- a/concepts/tagging/index.html
+++ b/concepts/tagging/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4477,6 +4494,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4494,7 +4531,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4510,14 +4547,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4583,7 +4620,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4599,14 +4636,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4662,7 +4699,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4678,14 +4715,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/concepts/tasks/index.html b/concepts/tasks/index.html
index e0eb5bd..3e4a377 100644
--- a/concepts/tasks/index.html
+++ b/concepts/tasks/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4534,6 +4551,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4551,7 +4588,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4567,14 +4604,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4640,7 +4677,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4656,14 +4693,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4719,7 +4756,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4735,14 +4772,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/OpenML-Docs/index.html b/contributing/OpenML-Docs/index.html
index 48fc461..48c3a26 100644
--- a/contributing/OpenML-Docs/index.html
+++ b/contributing/OpenML-Docs/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4546,6 +4563,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4563,7 +4600,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4579,14 +4616,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4731,7 +4768,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4747,14 +4784,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/Style/index.html b/contributing/Style/index.html
index 9b0664e..6c91051 100644
--- a/contributing/Style/index.html
+++ b/contributing/Style/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4523,6 +4540,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4540,7 +4577,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4556,14 +4593,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4629,7 +4666,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4645,14 +4682,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4708,7 +4745,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4724,14 +4761,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/backend/API-development/index.html b/contributing/backend/API-development/index.html
index 5abe48b..274f670 100644
--- a/contributing/backend/API-development/index.html
+++ b/contributing/backend/API-development/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4680,6 +4697,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4697,7 +4734,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4713,14 +4750,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4786,7 +4823,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4802,14 +4839,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4865,7 +4902,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4881,14 +4918,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/backend/Datasets/index.html b/contributing/backend/Datasets/index.html
index 3e40246..7741b23 100644
--- a/contributing/backend/Datasets/index.html
+++ b/contributing/backend/Datasets/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4545,6 +4562,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4562,7 +4599,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4578,14 +4615,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4651,7 +4688,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4667,14 +4704,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4730,7 +4767,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4746,14 +4783,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/backend/Java-App/index.html b/contributing/backend/Java-App/index.html
index a6f1e2a..2de78ba 100644
--- a/contributing/backend/Java-App/index.html
+++ b/contributing/backend/Java-App/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4545,6 +4562,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4562,7 +4599,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4578,14 +4615,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4651,7 +4688,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4667,14 +4704,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4730,7 +4767,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4746,14 +4783,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/backend/Local-Installation/index.html b/contributing/backend/Local-Installation/index.html
index 10c202f..b347f56 100644
--- a/contributing/backend/Local-Installation/index.html
+++ b/contributing/backend/Local-Installation/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4572,6 +4589,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4589,7 +4626,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4605,14 +4642,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4678,7 +4715,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4694,14 +4731,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4757,7 +4794,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4773,14 +4810,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/clients/Client-API-Standards/index.html b/contributing/clients/Client-API-Standards/index.html
index b836a29..fe06d99 100644
--- a/contributing/clients/Client-API-Standards/index.html
+++ b/contributing/clients/Client-API-Standards/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4569,6 +4586,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4586,7 +4623,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4602,14 +4639,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4675,7 +4712,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4691,14 +4728,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4754,7 +4791,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4770,14 +4807,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/clients/Rest/index.html b/contributing/clients/Rest/index.html
index 825250a..56c604f 100644
--- a/contributing/clients/Rest/index.html
+++ b/contributing/clients/Rest/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4576,6 +4593,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4593,7 +4630,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4609,14 +4646,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4682,7 +4719,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4698,14 +4735,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4761,7 +4798,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4777,14 +4814,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/clients/metadata_definition/index.html b/contributing/clients/metadata_definition/index.html
index 05994f3..6285ce9 100644
--- a/contributing/clients/metadata_definition/index.html
+++ b/contributing/clients/metadata_definition/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4578,6 +4595,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4595,7 +4632,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4611,14 +4648,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4684,7 +4721,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4700,14 +4737,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4763,7 +4800,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4779,14 +4816,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/index.html b/contributing/index.html
index e70db0d..f2c988a 100644
--- a/contributing/index.html
+++ b/contributing/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4465,6 +4482,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4482,7 +4519,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4498,14 +4535,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4650,7 +4687,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4666,14 +4703,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5073,6 +5110,29 @@ <h2 id="contributors-bot">Contributors bot<a class="headerlink" href="#contribut
     
     
     
+      
+  <span class="md-source-file__fact">
+    
+      
+  <span class="md-icon" title="Contributors">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2"/></svg>
+  </span>
+  <span>GitHub</span>
+
+    
+    <nav>
+      
+        <a href="https://github.com/joaquinvanschoren" class="md-author" title="@joaquinvanschoren">
+          
+          <img src="https://avatars.githubusercontent.com/u/1724503?v=4&size=72" alt="joaquinvanschoren">
+        </a>
+      
+      
+      
+    </nav>
+  </span>
+
+    
   </aside>
 
 
diff --git a/contributing/resources/index.html b/contributing/resources/index.html
index 28eb07a..a43cd45 100644
--- a/contributing/resources/index.html
+++ b/contributing/resources/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4532,6 +4549,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4549,7 +4586,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4565,14 +4602,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4638,7 +4675,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4654,14 +4691,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4717,7 +4754,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4733,14 +4770,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/website/Dash/index.html b/contributing/website/Dash/index.html
index 3d04c87..9bd0a4a 100644
--- a/contributing/website/Dash/index.html
+++ b/contributing/website/Dash/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4477,6 +4494,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4494,7 +4531,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4510,14 +4547,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4583,7 +4620,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4599,14 +4636,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4662,7 +4699,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4678,14 +4715,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/website/Flask/index.html b/contributing/website/Flask/index.html
index 062211b..6fc5d62 100644
--- a/contributing/website/Flask/index.html
+++ b/contributing/website/Flask/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4584,6 +4601,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4601,7 +4638,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4617,14 +4654,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4690,7 +4727,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4706,14 +4743,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4769,7 +4806,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4785,14 +4822,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/website/React/index.html b/contributing/website/React/index.html
index 5c95689..2e24c96 100644
--- a/contributing/website/React/index.html
+++ b/contributing/website/React/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4624,6 +4641,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4641,7 +4678,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4657,14 +4694,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4730,7 +4767,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4746,14 +4783,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4809,7 +4846,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4825,14 +4862,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/contributing/website/Website/index.html b/contributing/website/Website/index.html
index 5bf615f..47d900e 100644
--- a/contributing/website/Website/index.html
+++ b/contributing/website/Website/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4539,6 +4556,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4556,7 +4593,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4572,14 +4609,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4645,7 +4682,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4661,14 +4698,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4724,7 +4761,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4740,14 +4777,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/Java/index.html b/ecosystem/Java/index.html
index ef7a9fe..a5f3b80 100644
--- a/ecosystem/Java/index.html
+++ b/ecosystem/Java/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4849,6 +4866,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4866,7 +4903,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4882,14 +4919,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4955,7 +4992,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4971,14 +5008,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -5034,7 +5071,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -5050,14 +5087,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/MOA/index.html b/ecosystem/MOA/index.html
index 78ec3cf..bf3318a 100644
--- a/ecosystem/MOA/index.html
+++ b/ecosystem/MOA/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4510,6 +4527,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4527,7 +4564,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4543,14 +4580,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4616,7 +4653,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4632,14 +4669,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4695,7 +4732,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4711,14 +4748,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/Python_extensions/index.html b/ecosystem/Python_extensions/index.html
index b4de271..a451098 100644
--- a/ecosystem/Python_extensions/index.html
+++ b/ecosystem/Python_extensions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4571,6 +4588,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4588,7 +4625,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4604,14 +4641,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4677,7 +4714,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4693,14 +4730,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4756,7 +4793,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4772,14 +4809,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/Rest/index.html b/ecosystem/Rest/index.html
index c72934a..cba927d 100644
--- a/ecosystem/Rest/index.html
+++ b/ecosystem/Rest/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4574,6 +4591,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4591,7 +4628,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4607,14 +4644,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4680,7 +4717,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4696,14 +4733,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4759,7 +4796,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4775,14 +4812,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/Scikit-learn/basic_tutorial/index.html b/ecosystem/Scikit-learn/basic_tutorial/index.html
index afc54e1..22640ac 100644
--- a/ecosystem/Scikit-learn/basic_tutorial/index.html
+++ b/ecosystem/Scikit-learn/basic_tutorial/index.html
@@ -368,6 +368,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4472,6 +4489,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4489,7 +4526,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4505,14 +4542,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4578,7 +4615,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4594,14 +4631,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4657,7 +4694,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4673,14 +4710,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/Scikit-learn/datasets_tutorial/index.html b/ecosystem/Scikit-learn/datasets_tutorial/index.html
index fc69cd0..e37769a 100644
--- a/ecosystem/Scikit-learn/datasets_tutorial/index.html
+++ b/ecosystem/Scikit-learn/datasets_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4575,6 +4592,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4592,7 +4629,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4608,14 +4645,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4681,7 +4718,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4697,14 +4734,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4760,7 +4797,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4776,14 +4813,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/Scikit-learn/index.html b/ecosystem/Scikit-learn/index.html
index 3f836d1..b6db294 100644
--- a/ecosystem/Scikit-learn/index.html
+++ b/ecosystem/Scikit-learn/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4467,6 +4484,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4484,7 +4521,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4500,14 +4537,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4573,7 +4610,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4589,14 +4626,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/Weka/index.html b/ecosystem/Weka/index.html
index 8570911..c056d6c 100644
--- a/ecosystem/Weka/index.html
+++ b/ecosystem/Weka/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4546,6 +4563,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4563,7 +4600,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4579,14 +4616,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4731,7 +4768,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4747,14 +4784,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/ecosystem/index.html b/ecosystem/index.html
index 3bdb459..7cfe684 100644
--- a/ecosystem/index.html
+++ b/ecosystem/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4465,6 +4482,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4482,7 +4519,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4498,14 +4535,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4650,7 +4687,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4666,14 +4703,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5019,6 +5056,29 @@ <h2><a href="https://github.com/openml/flow-visualization">flow-visualization <i
     
     
     
+      
+  <span class="md-source-file__fact">
+    
+      
+  <span class="md-icon" title="Contributors">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2"/></svg>
+  </span>
+  <span>GitHub</span>
+
+    
+    <nav>
+      
+        <a href="https://github.com/joaquinvanschoren" class="md-author" title="@joaquinvanschoren">
+          
+          <img src="https://avatars.githubusercontent.com/u/1724503?v=4&size=72" alt="joaquinvanschoren">
+        </a>
+      
+      
+      
+    </nav>
+  </span>
+
+    
   </aside>
 
 
diff --git a/ecosystem/mlr/index.html b/ecosystem/mlr/index.html
index 036fb64..0e08e22 100644
--- a/ecosystem/mlr/index.html
+++ b/ecosystem/mlr/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4475,6 +4492,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4492,7 +4529,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4508,14 +4545,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4581,7 +4618,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4597,14 +4634,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4660,7 +4697,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4676,14 +4713,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/examples/20_basic/introduction_tutorial/index.html b/examples/20_basic/introduction_tutorial/index.html
index 3f838b6..e62b178 100644
--- a/examples/20_basic/introduction_tutorial/index.html
+++ b/examples/20_basic/introduction_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5297,7 +5334,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a116d10f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9b9a99ee">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5308,7 +5345,7 @@ <h1 id="introduction-tutorial-setup">Introduction tutorial &amp; Setup<a class="
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=019dfea8">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d082ee4b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5328,7 +5365,7 @@ <h1 id="introduction-tutorial-setup">Introduction tutorial &amp; Setup<a class="
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ba006b92">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=57fc2627">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5341,7 +5378,7 @@ <h1 id="installation">Installation<a class="anchor-link" href="#installation">¶
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f5556ddb">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=440e00f0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5371,7 +5408,7 @@ <h1 id="authentication">Authentication<a class="anchor-link" href="#authenticati
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6df0a0e4">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7468dbef">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5401,7 +5438,7 @@ <h1 id="authentication">Authentication<a class="anchor-link" href="#authenticati
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6a8dea7f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4a9d01e1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5413,7 +5450,7 @@ <h1 id="authentication">Authentication<a class="anchor-link" href="#authenticati
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=479e65e4">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8e38269c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5440,7 +5477,7 @@ <h1 id="authentication">Authentication<a class="anchor-link" href="#authenticati
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1e328c58">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=44a3a76f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5453,7 +5490,7 @@ <h1 id="authentication">Authentication<a class="anchor-link" href="#authenticati
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8d6aa0d0">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ab787d82">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5480,7 +5517,7 @@ <h1 id="authentication">Authentication<a class="anchor-link" href="#authenticati
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=fae3cdff">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ca63bdad">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5500,7 +5537,7 @@ <h1 id="caching">Caching<a class="anchor-link" href="#caching">¶</a></h1><p>Whe
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4ee1400b">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=56871948">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5531,7 +5568,7 @@ <h1 id="caching">Caching<a class="anchor-link" href="#caching">¶</a></h1><p>Whe
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=be933848">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=be603d2d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5542,7 +5579,7 @@ <h1 id="simple-example">Simple Example<a class="anchor-link" href="#simple-examp
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=65869135">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=701e8d56">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5585,7 +5622,7 @@ <h1 id="simple-example">Simple Example<a class="anchor-link" href="#simple-examp
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5370b305">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0d868fb5">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/20_basic/simple_datasets_tutorial/index.html b/examples/20_basic/simple_datasets_tutorial/index.html
index 646d57c..440bc2e 100644
--- a/examples/20_basic/simple_datasets_tutorial/index.html
+++ b/examples/20_basic/simple_datasets_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4536,6 +4553,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4553,7 +4590,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4569,14 +4606,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4642,7 +4679,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4658,14 +4695,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4721,7 +4758,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4737,14 +4774,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5389,7 +5426,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=347db721">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=22081fd1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5404,7 +5441,7 @@ <h1 id="datasets">Datasets<a class="anchor-link" href="#datasets">¶</a></h1><p>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=25f2eb6c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=72640ac1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5432,7 +5469,7 @@ <h1 id="datasets">Datasets<a class="anchor-link" href="#datasets">¶</a></h1><p>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f08c96a6">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5113ee96">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5443,7 +5480,7 @@ <h2 id="list-datasets">List datasets<a class="anchor-link" href="#list-datasets"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=732c6181">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f60065dd">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5472,7 +5509,7 @@ <h2 id="list-datasets">List datasets<a class="anchor-link" href="#list-datasets"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=91d7a484">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=56e03e79">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5483,7 +5520,7 @@ <h2 id="download-a-dataset">Download a dataset<a class="anchor-link" href="#down
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a1f99d61">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e229b0c2">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5528,7 +5565,7 @@ <h2 id="download-a-dataset">Download a dataset<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6c55d005">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c8cce235">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5544,7 +5581,7 @@ <h2 id="load-a-dataset">Load a dataset<a class="anchor-link" href="#load-a-datas
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e10de345">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b4ac9753">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5575,7 +5612,7 @@ <h2 id="load-a-dataset">Load a dataset<a class="anchor-link" href="#load-a-datas
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=71693b44">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c9459954">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5586,7 +5623,7 @@ <h2 id="load-a-dataset">Load a dataset<a class="anchor-link" href="#load-a-datas
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=55566871">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1ec88636">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/20_basic/simple_flows_and_runs_tutorial/index.html b/examples/20_basic/simple_flows_and_runs_tutorial/index.html
index d339f47..4ee0c4c 100644
--- a/examples/20_basic/simple_flows_and_runs_tutorial/index.html
+++ b/examples/20_basic/simple_flows_and_runs_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4536,6 +4553,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4553,7 +4590,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4569,14 +4606,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4642,7 +4679,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4658,14 +4695,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4721,7 +4758,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4737,14 +4774,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5389,7 +5426,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0547e9ee">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=71ca83f6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5400,7 +5437,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9ed899f2">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6954666d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5429,7 +5466,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=017e5423">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2dcdbe22">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5441,7 +5478,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=573052bd">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8a5242e9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5468,7 +5505,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6f60e98f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1e96250b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5479,7 +5516,7 @@ <h2 id="train-a-machine-learning-model">Train a machine learning model<a class="
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9484f5de">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1fdeefce">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5516,7 +5553,7 @@ <h2 id="train-a-machine-learning-model">Train a machine learning model<a class="
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f2d7ce5a">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9a99d818">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5527,7 +5564,7 @@ <h2 id="running-a-model-on-a-task">Running a model on a task<a class="anchor-lin
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4c363b9f">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f90bdbed">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5560,7 +5597,7 @@ <h2 id="running-a-model-on-a-task">Running a model on a task<a class="anchor-lin
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f080192a">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3295d0e5">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5571,7 +5608,7 @@ <h2 id="publishing-the-run">Publishing the run<a class="anchor-link" href="#publ
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=448acb08">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=be3c1d40">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5602,7 +5639,7 @@ <h2 id="publishing-the-run">Publishing the run<a class="anchor-link" href="#publ
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=cea80828">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4d0febf5">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/20_basic/simple_suites_tutorial/index.html b/examples/20_basic/simple_suites_tutorial/index.html
index ed41783..e1d319f 100644
--- a/examples/20_basic/simple_suites_tutorial/index.html
+++ b/examples/20_basic/simple_suites_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5297,7 +5334,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5e24f393">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=079fd850">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5311,7 +5348,7 @@ <h1 id="benchmark-suites">Benchmark suites<a class="anchor-link" href="#benchmar
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=583c70d2">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=949b64b9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5338,7 +5375,7 @@ <h1 id="benchmark-suites">Benchmark suites<a class="anchor-link" href="#benchmar
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=13561cbd">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c9fdc9a9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5364,7 +5401,7 @@ <h1 id="openml-cc18">OpenML-CC18<a class="anchor-link" href="#openml-cc18">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a930bec7">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=348727c7">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5375,7 +5412,7 @@ <h1 id="downloading-benchmark-suites">Downloading benchmark suites<a class="anch
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e2fea568">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=05b7e3f8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5404,7 +5441,7 @@ <h1 id="downloading-benchmark-suites">Downloading benchmark suites<a class="anch
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=97655305">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b3bb56e9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5417,7 +5454,7 @@ <h1 id="downloading-benchmark-suites">Downloading benchmark suites<a class="anch
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5e2e00c8">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ea5dd218">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5446,7 +5483,7 @@ <h1 id="downloading-benchmark-suites">Downloading benchmark suites<a class="anch
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bf0c36eb">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=588d7757">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5457,7 +5494,7 @@ <h1 id="downloading-benchmark-suites">Downloading benchmark suites<a class="anch
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1d84edc3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=10f5a9e6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5488,7 +5525,7 @@ <h1 id="downloading-benchmark-suites">Downloading benchmark suites<a class="anch
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=cf4a0258">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b23a8bdd">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/configure_logging/index.html b/examples/30_extended/configure_logging/index.html
index f386426..0c76cc8 100644
--- a/examples/30_extended/configure_logging/index.html
+++ b/examples/30_extended/configure_logging/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5297,7 +5334,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bd71999a">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4fcd2bba">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5319,7 +5356,7 @@ <h1 id="logging">Logging<a class="anchor-link" href="#logging">¶</a></h1><p>Thi
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6ad9a76b">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2bfec0d0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5350,7 +5387,7 @@ <h1 id="logging">Logging<a class="anchor-link" href="#logging">¶</a></h1><p>Thi
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f529c386">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3b8488bd">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5368,7 +5405,7 @@ <h1 id="logging">Logging<a class="anchor-link" href="#logging">¶</a></h1><p>Thi
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8803d5e8">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a48763ae">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5403,7 +5440,7 @@ <h1 id="logging">Logging<a class="anchor-link" href="#logging">¶</a></h1><p>Thi
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=de4d971c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0567c3cc">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/create_upload_tutorial/index.html b/examples/30_extended/create_upload_tutorial/index.html
index 89a95ec..7d3bffc 100644
--- a/examples/30_extended/create_upload_tutorial/index.html
+++ b/examples/30_extended/create_upload_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5297,7 +5334,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=86e2a36e">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bada8507">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5308,7 +5345,7 @@ <h1 id="dataset-upload-tutorial">Dataset upload tutorial<a class="anchor-link" h
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a19be1b4">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b9338b43">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5347,7 +5384,7 @@ <h1 id="dataset-upload-tutorial">Dataset upload tutorial<a class="anchor-link" h
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6795b208">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d231d404">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5359,7 +5396,7 @@ <h1 id="dataset-upload-tutorial">Dataset upload tutorial<a class="anchor-link" h
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=76baf1fa">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=630ab6a9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5386,7 +5423,7 @@ <h1 id="dataset-upload-tutorial">Dataset upload tutorial<a class="anchor-link" h
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c94da474">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=064bad07">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5404,7 +5441,7 @@ <h1 id="dataset-upload-tutorial">Dataset upload tutorial<a class="anchor-link" h
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a08d678e">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bed66db7">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5418,7 +5455,7 @@ <h1 id="prepare-dataset">Prepare dataset<a class="anchor-link" href="#prepare-da
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f231e181">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6a87c26a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5455,7 +5492,7 @@ <h1 id="prepare-dataset">Prepare dataset<a class="anchor-link" href="#prepare-da
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=dfa94006">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9fc01dab">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5469,7 +5506,7 @@ <h1 id="prepare-dataset">Prepare dataset<a class="anchor-link" href="#prepare-da
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a121b583">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=13dd3d7a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5516,7 +5553,7 @@ <h1 id="prepare-dataset">Prepare dataset<a class="anchor-link" href="#prepare-da
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=fb8cde7c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ad18d0cf">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5529,7 +5566,7 @@ <h1 id="create-the-dataset-object">Create the dataset object<a class="anchor-lin
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a0125a78">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=684ab958">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5626,7 +5663,7 @@ <h1 id="create-the-dataset-object">Create the dataset object<a class="anchor-lin
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8a4b344f">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1d77a71d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5656,7 +5693,7 @@ <h1 id="create-the-dataset-object">Create the dataset object<a class="anchor-lin
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b8bdc22a">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3975ce13">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5670,7 +5707,7 @@ <h2 id="dataset-is-a-list">Dataset is a list<a class="anchor-link" href="#datase
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=42cee3fd">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=61829bef">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5811,7 +5848,7 @@ <h2 id="dataset-is-a-list">Dataset is a list<a class="anchor-link" href="#datase
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f05ffed3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c3526f3b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5840,7 +5877,7 @@ <h2 id="dataset-is-a-list">Dataset is a list<a class="anchor-link" href="#datase
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=96389395">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=82896219">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5857,7 +5894,7 @@ <h2 id="dataset-is-a-pandas-dataframe">Dataset is a pandas DataFrame<a class="an
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ef53ed42">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7606dd00">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5896,7 +5933,7 @@ <h2 id="dataset-is-a-pandas-dataframe">Dataset is a pandas DataFrame<a class="an
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c6c48685">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7fd86ee8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5911,7 +5948,7 @@ <h2 id="dataset-is-a-pandas-dataframe">Dataset is a pandas DataFrame<a class="an
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=bf468516">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a83c903a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5968,7 +6005,7 @@ <h2 id="dataset-is-a-pandas-dataframe">Dataset is a pandas DataFrame<a class="an
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ff0de896">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8d0fe2c6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5997,7 +6034,7 @@ <h2 id="dataset-is-a-pandas-dataframe">Dataset is a pandas DataFrame<a class="an
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d696879b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bdd0986b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6008,7 +6045,7 @@ <h1 id="dataset-is-a-sparse-matrix">Dataset is a sparse matrix<a class="anchor-l
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9fdd76ad">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=39db752f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6085,7 +6122,7 @@ <h1 id="dataset-is-a-sparse-matrix">Dataset is a sparse matrix<a class="anchor-l
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=644de25a">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c18a2f59">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6114,7 +6151,7 @@ <h1 id="dataset-is-a-sparse-matrix">Dataset is a sparse matrix<a class="anchor-l
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a4ed3435">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2db7f2c9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6146,7 +6183,7 @@ <h2 id="dataset-is-a-pandas-dataframe-with-sparse-columns">Dataset is a pandas d
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4f1ed7f6">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=239471a1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6176,7 +6213,7 @@ <h2 id="dataset-is-a-pandas-dataframe-with-sparse-columns">Dataset is a pandas d
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=328b29e9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5e07adb6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/custom_flow_/index.html b/examples/30_extended/custom_flow_/index.html
index 028e4e2..f27735e 100644
--- a/examples/30_extended/custom_flow_/index.html
+++ b/examples/30_extended/custom_flow_/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4527,6 +4544,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4544,7 +4581,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4560,14 +4597,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4633,7 +4670,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4649,14 +4686,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4712,7 +4749,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4728,14 +4765,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5371,7 +5408,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f7d0d9a3">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=313c70ef">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5390,7 +5427,7 @@ <h1 id="creating-and-using-a-custom-flow">Creating and Using a Custom Flow<a cla
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=cb88c1b5">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=21e76492">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5427,7 +5464,7 @@ <h1 id="creating-and-using-a-custom-flow">Creating and Using a Custom Flow<a cla
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d357bf15">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2d7dd30d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5439,7 +5476,7 @@ <h1 id="creating-and-using-a-custom-flow">Creating and Using a Custom Flow<a cla
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7f95eb39">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=bf57dc5c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5466,7 +5503,7 @@ <h1 id="creating-and-using-a-custom-flow">Creating and Using a Custom Flow<a cla
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0cb19b4b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=67a76b88">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5486,7 +5523,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=22c0ddeb">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=16a05ff4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5533,7 +5570,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=309e675e">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1a34ff59">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5547,7 +5584,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a5a5e0db">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3a371dce">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5588,7 +5625,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c9ff7a27">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=dc197c02">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5609,7 +5646,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d73b9035">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d95e6ca6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5646,7 +5683,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9654be02">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8f05e400">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5663,7 +5700,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=71bf06b5">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e2b06d8c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5704,7 +5741,7 @@ <h2 id="1-defining-the-flow">1. Defining the flow<a class="anchor-link" href="#1
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=20310042">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c2459839">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5718,7 +5755,7 @@ <h2 id="2-using-the-flow">2. Using the flow<a class="anchor-link" href="#2-using
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5bdbec02">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3105286f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5765,7 +5802,7 @@ <h2 id="2-using-the-flow">2. Using the flow<a class="anchor-link" href="#2-using
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d42d6703">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9dbe7113">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5796,7 +5833,7 @@ <h2 id="2-using-the-flow">2. Using the flow<a class="anchor-link" href="#2-using
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=98c4e275">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3cd07270">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5889,7 +5926,7 @@ <h2 id="2-using-the-flow">2. Using the flow<a class="anchor-link" href="#2-using
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e7eb93a0">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=96fb60a2">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5901,7 +5938,7 @@ <h2 id="2-using-the-flow">2. Using the flow<a class="anchor-link" href="#2-using
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=15934bc7">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f8537bb8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5952,7 +5989,7 @@ <h2 id="2-using-the-flow">2. Using the flow<a class="anchor-link" href="#2-using
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8094fed9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d435ccae">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/datasets_tutorial/index.html b/examples/30_extended/datasets_tutorial/index.html
index aea42ea..2e1d941 100644
--- a/examples/30_extended/datasets_tutorial/index.html
+++ b/examples/30_extended/datasets_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4563,6 +4580,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4580,7 +4617,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4596,14 +4633,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4669,7 +4706,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4685,14 +4722,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4748,7 +4785,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4764,14 +4801,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5443,7 +5480,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7ad24712">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=27c0ace8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5454,7 +5491,7 @@ <h1 id="datasets">Datasets<a class="anchor-link" href="#datasets">¶</a></h1><p>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=50fa5c51">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1a730f59">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5485,7 +5522,7 @@ <h1 id="datasets">Datasets<a class="anchor-link" href="#datasets">¶</a></h1><p>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9aab715f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=576ac078">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5498,7 +5535,7 @@ <h2 id="exercise-0">Exercise 0<a class="anchor-link" href="#exercise-0">¶</a></
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e4ea42d0">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4860dbab">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5541,7 +5578,7 @@ <h2 id="exercise-0">Exercise 0<a class="anchor-link" href="#exercise-0">¶</a></
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3ecbf461">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1c2d1cd5">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5556,7 +5593,7 @@ <h2 id="exercise-1">Exercise 1<a class="anchor-link" href="#exercise-1">¶</a></
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e28f0e93">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ec11b765">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5583,7 +5620,7 @@ <h2 id="exercise-1">Exercise 1<a class="anchor-link" href="#exercise-1">¶</a></
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=946bcb38">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f3e84b25">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5610,7 +5647,7 @@ <h2 id="exercise-1">Exercise 1<a class="anchor-link" href="#exercise-1">¶</a></
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=54b50109">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1341c705">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5637,7 +5674,7 @@ <h2 id="exercise-1">Exercise 1<a class="anchor-link" href="#exercise-1">¶</a></
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=41726022">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=22004846">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5648,7 +5685,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ba79f856">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8e20aad8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5693,7 +5730,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b38ee85d">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d041e393">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5706,7 +5743,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=942e5dfa">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=44457af1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5733,7 +5770,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e802a313">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b46ad0fd">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5746,7 +5783,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ed1a001a">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2267f6b6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5781,7 +5818,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3d172ca8">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4985d3ce">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5799,7 +5836,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a759187f">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b562dd8f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5826,7 +5863,7 @@ <h2 id="download-datasets">Download datasets<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e356c46c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=03a08ec0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5839,7 +5876,7 @@ <h2 id="exercise-2">Exercise 2<a class="anchor-link" href="#exercise-2">¶</a></
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d85d6849">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e58cd02d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5884,7 +5921,7 @@ <h2 id="exercise-2">Exercise 2<a class="anchor-link" href="#exercise-2">¶</a></
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=607cd76c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a70e1356">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5897,7 +5934,7 @@ <h2 id="edit-a-created-dataset">Edit a created dataset<a class="anchor-link" hre
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7aed36ce">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=683a2d13">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5924,7 +5961,7 @@ <h2 id="edit-a-created-dataset">Edit a created dataset<a class="anchor-link" hre
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=84134321">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8d1737b0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5937,7 +5974,7 @@ <h2 id="edit-a-created-dataset">Edit a created dataset<a class="anchor-link" hre
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f4990662">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=23058fb9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5994,7 +6031,7 @@ <h2 id="edit-a-created-dataset">Edit a created dataset<a class="anchor-link" hre
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ce525990">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=cd0f4e3d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6010,7 +6047,7 @@ <h2 id="edit-a-created-dataset">Edit a created dataset<a class="anchor-link" hre
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c0afac3c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3a4fd45f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6043,7 +6080,7 @@ <h2 id="edit-a-created-dataset">Edit a created dataset<a class="anchor-link" hre
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=68bb34c7">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=132eae31">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6057,7 +6094,7 @@ <h2 id="fork-dataset">Fork dataset<a class="anchor-link" href="#fork-dataset">¶
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=37e447c1">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=52120deb">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6090,7 +6127,7 @@ <h2 id="fork-dataset">Fork dataset<a class="anchor-link" href="#fork-dataset">¶
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2ecf847e">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=66e5c525">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/fetch_evaluations_tutorial/index.html b/examples/30_extended/fetch_evaluations_tutorial/index.html
index 8c12d86..6705085 100644
--- a/examples/30_extended/fetch_evaluations_tutorial/index.html
+++ b/examples/30_extended/fetch_evaluations_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4563,6 +4580,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4580,7 +4617,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4596,14 +4633,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4669,7 +4706,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4685,14 +4722,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4748,7 +4785,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4764,14 +4801,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5443,7 +5480,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=88184191">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b66bbdd1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5467,7 +5504,7 @@ <h1 id="fetching-evaluations">Fetching Evaluations<a class="anchor-link" href="#
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9974bdd5">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=18745fc6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5494,7 +5531,7 @@ <h1 id="fetching-evaluations">Fetching Evaluations<a class="anchor-link" href="#
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=11179cee">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=fb8c98e3">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5507,7 +5544,7 @@ <h2 id="listing-evaluations">Listing evaluations<a class="anchor-link" href="#li
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c57946a3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=170c6092">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5554,7 +5591,7 @@ <h2 id="listing-evaluations">Listing evaluations<a class="anchor-link" href="#li
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=dfc87be6">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0cd5df91">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5566,7 +5603,7 @@ <h2 id="viewing-a-sample-task">Viewing a sample task<a class="anchor-link" href=
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7191d623">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=dfa480e7">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5597,7 +5634,7 @@ <h2 id="viewing-a-sample-task">Viewing a sample task<a class="anchor-link" href=
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=31d1004b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6655c184">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5610,7 +5647,7 @@ <h2 id="obtaining-all-the-evaluations-for-the-task">Obtaining all the evaluation
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=44bd4ac4">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=74593336">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5655,7 +5692,7 @@ <h2 id="obtaining-all-the-evaluations-for-the-task">Obtaining all the evaluation
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ebf75008">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=96308aee">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5668,7 +5705,7 @@ <h2 id="obtaining-cdf-of-metric-for-chosen-task">Obtaining CDF of metric for cho
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9faf3a07">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=603fbb0e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5733,7 +5770,7 @@ <h2 id="obtaining-cdf-of-metric-for-chosen-task">Obtaining CDF of metric for cho
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4a991585">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=79e2aea8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5746,7 +5783,7 @@ <h2 id="obtaining-cdf-of-metric-for-chosen-task">Obtaining CDF of metric for cho
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ae01ab5d">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2b733e69">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5758,7 +5795,7 @@ <h2 id="comparing-top-10-performing-flows">Comparing top 10 performing flows<a c
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1fdc3fba">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=82bba631">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5847,7 +5884,7 @@ <h2 id="comparing-top-10-performing-flows">Comparing top 10 performing flows<a c
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1192ee37">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6d3a4c22">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5863,7 +5900,7 @@ <h2 id="comparing-top-10-performing-flows">Comparing top 10 performing flows<a c
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=932fc05a">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f4f540ab">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5898,7 +5935,7 @@ <h2 id="comparing-top-10-performing-flows">Comparing top 10 performing flows<a c
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a4867dcf">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9c135b24">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5911,7 +5948,7 @@ <h2 id="obtaining-evaluations-with-hyperparameter-settings">Obtaining evaluation
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c813aa16">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=07b5a637">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5946,7 +5983,7 @@ <h2 id="obtaining-evaluations-with-hyperparameter-settings">Obtaining evaluation
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7aeb5881">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=418ef274">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5959,7 +5996,7 @@ <h2 id="obtaining-evaluations-with-hyperparameter-settings">Obtaining evaluation
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7eb79913">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8cb4adf6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/fetch_runtimes_tutorial/index.html b/examples/30_extended/fetch_runtimes_tutorial/index.html
index 8dc9f03..a8a1dae 100644
--- a/examples/30_extended/fetch_runtimes_tutorial/index.html
+++ b/examples/30_extended/fetch_runtimes_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5297,7 +5334,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0e3647d9">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f2719158">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5356,7 +5393,7 @@
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4051491d">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=de60b1e0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5367,7 +5404,7 @@ <h1 id="preparing-tasks-and-scikit-learn-models">Preparing tasks and scikit-lear
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1ba5f49f">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=eb420762">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5444,7 +5481,7 @@ <h1 id="preparing-tasks-and-scikit-learn-models">Preparing tasks and scikit-lear
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f0aa6ecf">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6f9316cd">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5457,7 +5494,7 @@ <h1 id="case-1-running-a-random-forest-model-on-an-openml-task">Case 1: Running
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c9b6fd52">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b33f8d17">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5528,7 +5565,7 @@ <h1 id="case-1-running-a-random-forest-model-on-an-openml-task">Case 1: Running
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=63c412e6">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bcb0030b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5549,7 +5586,7 @@ <h1 id="case-1-running-a-random-forest-model-on-an-openml-task">Case 1: Running
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=981e1d0e">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=41cde056">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5576,7 +5613,7 @@ <h1 id="case-1-running-a-random-forest-model-on-an-openml-task">Case 1: Running
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7174966a">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=65dfbb8e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5587,7 +5624,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=66c6c0c3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=48db3347">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5628,7 +5665,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a7709b89">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7fa8ff82">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5639,7 +5676,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=14f3e245">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5e232ffa">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5678,7 +5715,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b86a3d04">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=48cb9b62">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5692,7 +5729,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=87b85af1">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=16644428">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5707,7 +5744,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5967143d">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b2c67bc8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5721,7 +5758,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=76f5b5b3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8ecc5974">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5758,7 +5795,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=cd37b290">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bcc18f0d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5775,7 +5812,7 @@ <h2 id="case-2-running-scikit-learn-model-on-an-openml-task-in-parallel">Case 2:
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a610e253">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c3ba2fd8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5791,7 +5828,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a0f2bc06">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=fea27819">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5852,7 +5889,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6acc1f6f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=09201f66">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5873,7 +5910,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1d60cf9d">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=dc9010bb">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5908,7 +5945,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f3e59f16">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=37d4e7a7">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5921,7 +5958,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c24ef953">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ebb3f231">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5976,7 +6013,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=27ea458f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a328a52d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5995,7 +6032,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=22dc57c5">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7462f565">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6053,7 +6090,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3d7a3328">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1822c5c8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6067,7 +6104,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=577e6d72">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c2da57de">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6118,7 +6155,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a699caef">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=725228ab">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6138,7 +6175,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8b4b61ca">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8e0b531d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6245,7 +6282,7 @@ <h2 id="case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes">Case
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=87ce3dc8">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=da93939f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6260,7 +6297,7 @@ <h2 id="case-4-running-models-that-scikit-learn-doesnt-parallelize">Case 4: Runn
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=45860dd9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2efcef28">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6299,7 +6336,7 @@ <h2 id="case-4-running-models-that-scikit-learn-doesnt-parallelize">Case 4: Runn
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=876c1cb4">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=abb8f6ed">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6312,7 +6349,7 @@ <h2 id="case-4-running-models-that-scikit-learn-doesnt-parallelize">Case 4: Runn
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a1d0bb3b">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ac125b78">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6349,7 +6386,7 @@ <h2 id="case-4-running-models-that-scikit-learn-doesnt-parallelize">Case 4: Runn
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=930ffa3b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0b9b43d5">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6362,7 +6399,7 @@ <h2 id="case-4-running-models-that-scikit-learn-doesnt-parallelize">Case 4: Runn
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7ad0c117">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4deac6c3">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6401,7 +6438,7 @@ <h2 id="case-4-running-models-that-scikit-learn-doesnt-parallelize">Case 4: Runn
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2cdb636f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9ddcc2a6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6416,7 +6453,7 @@ <h2 id="case-5-running-scikit-learn-models-that-dont-release-gil">Case 5: Runnin
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=11713f57">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7bf82701">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6457,7 +6494,7 @@ <h2 id="case-5-running-scikit-learn-models-that-dont-release-gil">Case 5: Runnin
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=cdf54ba8">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a178a96c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/flow_id_tutorial/index.html b/examples/30_extended/flow_id_tutorial/index.html
index 107609d..99570f2 100644
--- a/examples/30_extended/flow_id_tutorial/index.html
+++ b/examples/30_extended/flow_id_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4527,6 +4544,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4544,7 +4581,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4560,14 +4597,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4633,7 +4670,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4649,14 +4686,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4712,7 +4749,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4728,14 +4765,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5371,7 +5408,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e33ce553">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e11112d4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5383,7 +5420,7 @@ <h1 id="obtaining-flow-ids">Obtaining Flow IDs<a class="anchor-link" href="#obta
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a68cdcc0">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f25740c2">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5414,7 +5451,7 @@ <h1 id="obtaining-flow-ids">Obtaining Flow IDs<a class="anchor-link" href="#obta
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b2cbe1ae">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7e26bef9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5426,7 +5463,7 @@ <h1 id="obtaining-flow-ids">Obtaining Flow IDs<a class="anchor-link" href="#obta
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=60c80ff9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=86559efe">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5453,7 +5490,7 @@ <h1 id="obtaining-flow-ids">Obtaining Flow IDs<a class="anchor-link" href="#obta
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c5100d88">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=eed55f8c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5482,7 +5519,7 @@ <h1 id="obtaining-flow-ids">Obtaining Flow IDs<a class="anchor-link" href="#obta
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ebc92688">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c26c8179">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5493,7 +5530,7 @@ <h2 id="1-obtaining-a-flow-given-a-classifier">1. Obtaining a flow given a class
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6840eff0">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=09f8e8c4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5524,7 +5561,7 @@ <h2 id="1-obtaining-a-flow-given-a-classifier">1. Obtaining a flow given a class
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7dc84df4">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=748e41e4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5542,7 +5579,7 @@ <h2 id="1-obtaining-a-flow-given-a-classifier">1. Obtaining a flow given a class
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=43a4f5fa">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c56c268a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5571,7 +5608,7 @@ <h2 id="1-obtaining-a-flow-given-a-classifier">1. Obtaining a flow given a class
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=da226fb3">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=afe2239a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5588,7 +5625,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=23d7c64a">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=30f7257b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5615,7 +5652,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=63439f63">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0d577911">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5627,7 +5664,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=09a33c2f">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f4c758ba">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5656,7 +5693,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ce358caa">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7f7b9b34">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5667,7 +5704,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ee80c0ac">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=bd1996ab">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5696,7 +5733,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=46b34eb9">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=036e46d8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5707,7 +5744,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b66094c5">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6138bb31">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5736,7 +5773,7 @@ <h2 id="2-obtaining-a-flow-given-its-name">2. Obtaining a flow given its name<a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=78811722">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9e4e25f4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/flows_and_runs_tutorial/index.html b/examples/30_extended/flows_and_runs_tutorial/index.html
index 837ff74..7b47c67 100644
--- a/examples/30_extended/flows_and_runs_tutorial/index.html
+++ b/examples/30_extended/flows_and_runs_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4554,6 +4571,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4660,7 +4697,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4676,14 +4713,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4739,7 +4776,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4755,14 +4792,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5425,7 +5462,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=742c7324">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=25e8a5a3">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5436,7 +5473,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=62e2dec2">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=83ac30df">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5465,7 +5502,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=328c80b5">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1a7399de">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5478,7 +5515,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=719f54db">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7a7dd73c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5505,7 +5542,7 @@ <h1 id="flows-and-runs">Flows and Runs<a class="anchor-link" href="#flows-and-ru
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1a4c8344">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c8c8a31f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5516,7 +5553,7 @@ <h2 id="train-machine-learning-models">Train machine learning models<a class="an
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1960d8de">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=dc7e83c5">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5555,7 +5592,7 @@ <h2 id="train-machine-learning-models">Train machine learning models<a class="an
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=23065797">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b74b0e4e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5569,7 +5606,7 @@ <h2 id="train-machine-learning-models">Train machine learning models<a class="an
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=687c99f0">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f8c03584">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5614,7 +5651,7 @@ <h2 id="train-machine-learning-models">Train machine learning models<a class="an
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=078746a5">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=54f023d8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5625,7 +5662,7 @@ <h2 id="runs-easily-explore-models">Runs: Easily explore models<a class="anchor-
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=dff6bc2c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6fa1aed6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5670,7 +5707,7 @@ <h2 id="runs-easily-explore-models">Runs: Easily explore models<a class="anchor-
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ab01e570">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a247143c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5683,7 +5720,7 @@ <h2 id="runs-easily-explore-models">Runs: Easily explore models<a class="anchor-
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=677c79b4">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ef66264b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5716,7 +5753,7 @@ <h2 id="runs-easily-explore-models">Runs: Easily explore models<a class="anchor-
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=dfbdfbd5">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=30f30d5f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5727,7 +5764,7 @@ <h2 id="runs-easily-explore-models">Runs: Easily explore models<a class="anchor-
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=243d2f78">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d14b8ac4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5756,7 +5793,7 @@ <h2 id="runs-easily-explore-models">Runs: Easily explore models<a class="anchor-
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1ff6257b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=14563924">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5770,7 +5807,7 @@ <h2 id="it-also-works-with-pipelines">It also works with pipelines<a class="anch
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c0ca68b7">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f7165bca">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5857,7 +5894,7 @@ <h2 id="it-also-works-with-pipelines">It also works with pipelines<a class="anch
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d8f4b372">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0141fc4e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5870,7 +5907,7 @@ <h2 id="it-also-works-with-pipelines">It also works with pipelines<a class="anch
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=34027803">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7fee0821">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5971,7 +6008,7 @@ <h2 id="it-also-works-with-pipelines">It also works with pipelines<a class="anch
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a6518f64">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=80d20db1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5985,7 +6022,7 @@ <h2 id="running-flows-on-tasks-offline-for-later-upload">Running flows on tasks
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=12dc5e31">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b94c1ad1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6048,7 +6085,7 @@ <h2 id="running-flows-on-tasks-offline-for-later-upload">Running flows on tasks
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=937dfa0c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=44b41e33">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6059,7 +6096,7 @@ <h2 id="running-flows-on-tasks-offline-for-later-upload">Running flows on tasks
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=41325d9b">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ea181cfe">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6108,7 +6145,7 @@ <h2 id="running-flows-on-tasks-offline-for-later-upload">Running flows on tasks
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=19ff403d">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e932f92a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6133,7 +6170,7 @@ <h2 id="challenge">Challenge<a class="anchor-link" href="#challenge">¶</a></h2>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d4e88166">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ebe963d2">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6176,7 +6213,7 @@ <h2 id="challenge">Challenge<a class="anchor-link" href="#challenge">¶</a></h2>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b84d63fd">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6234b832">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/plot_svm_hyperparameters_tutorial/index.html b/examples/30_extended/plot_svm_hyperparameters_tutorial/index.html
index a92d53a..8e15623 100644
--- a/examples/30_extended/plot_svm_hyperparameters_tutorial/index.html
+++ b/examples/30_extended/plot_svm_hyperparameters_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5297,7 +5334,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a2e5f9ed">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6b970ca1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5308,7 +5345,7 @@ <h1 id="plotting-hyperparameter-surfaces">Plotting hyperparameter surfaces<a cla
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0fd282dd">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=85c6a27b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5337,7 +5374,7 @@ <h1 id="plotting-hyperparameter-surfaces">Plotting hyperparameter surfaces<a cla
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b6a8d400">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e09c355b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5352,7 +5389,7 @@ <h1 id="first-step-obtaining-the-data">First step - obtaining the data<a class="
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3775315f">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=70843954">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5397,7 +5434,7 @@ <h1 id="first-step-obtaining-the-data">First step - obtaining the data<a class="
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b6bb5bb9">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1735487e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5408,7 +5445,7 @@ <h1 id="first-step-obtaining-the-data">First step - obtaining the data<a class="
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4206d824">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=082646ef">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5437,7 +5474,7 @@ <h1 id="first-step-obtaining-the-data">First step - obtaining the data<a class="
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c98f47e9">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=859c36a8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5449,7 +5486,7 @@ <h1 id="first-step-obtaining-the-data">First step - obtaining the data<a class="
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d39a44f3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3450ea2a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5478,7 +5515,7 @@ <h1 id="first-step-obtaining-the-data">First step - obtaining the data<a class="
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=eaaf3755">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e5eb9a52">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5489,7 +5526,7 @@ <h2 id="option-1-plotting-via-the-pandas-helper-functions">Option 1 - plotting v
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3e74d7c9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b5aa4f0d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5530,7 +5567,7 @@ <h2 id="option-1-plotting-via-the-pandas-helper-functions">Option 1 - plotting v
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=01512d4b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=439d398f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5541,7 +5578,7 @@ <h2 id="option-2-plotting-via-matplotlib">Option 2 - plotting via matplotlib<a c
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f2265957">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b9fda4df">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/run_setup_tutorial/index.html b/examples/30_extended/run_setup_tutorial/index.html
index 5dd73eb..5959f3d 100644
--- a/examples/30_extended/run_setup_tutorial/index.html
+++ b/examples/30_extended/run_setup_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4527,6 +4544,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4544,7 +4581,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4560,14 +4597,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4633,7 +4670,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4649,14 +4686,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4712,7 +4749,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4728,14 +4765,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5371,7 +5408,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5a2ea5a2">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=be083bc7">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5398,7 +5435,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f22a950d">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4aa7e2aa">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5444,7 +5481,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c32cc8ee">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2add7f61">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5456,7 +5493,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a07ff2db">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=89bd901b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5483,7 +5520,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4d61539c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=724e1850">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5497,7 +5534,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=a3508940">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e6a9deb2">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5524,7 +5561,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9e4dfa35">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=226c6192">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5537,7 +5574,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6730dd85">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6f14b55d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5586,7 +5623,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f50f58a9">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=81f6f727">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5600,7 +5637,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5136269e">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6d5bb256">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5647,7 +5684,7 @@ <h1 id="run-setup">Run Setup<a class="anchor-link" href="#run-setup">¶</a></h1>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ec42960c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4c51a31f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5658,7 +5695,7 @@ <h2 id="2-download-the-flow-and-solve-the-same-task-again">2) Download the flow
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=390ca563">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=255ca41b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5705,7 +5742,7 @@ <h2 id="2-download-the-flow-and-solve-the-same-task-again">2) Download the flow
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=470ece2e">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=39839fd1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5716,7 +5753,7 @@ <h2 id="3-we-will-verify-that-the-obtained-results-are-exactly-the-same">3) We w
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=22e67ab3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b513bf23">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5745,7 +5782,7 @@ <h2 id="3-we-will-verify-that-the-obtained-results-are-exactly-the-same">3) We w
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ac9db136">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5faa69ac">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/study_tutorial/index.html b/examples/30_extended/study_tutorial/index.html
index 4d34962..567ea1b 100644
--- a/examples/30_extended/study_tutorial/index.html
+++ b/examples/30_extended/study_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4536,6 +4553,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4553,7 +4590,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4569,14 +4606,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4642,7 +4679,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4658,14 +4695,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4721,7 +4758,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4737,14 +4774,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5389,7 +5426,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8f6cfa15">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=620fef29">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5404,7 +5441,7 @@ <h1 id="benchmark-studies">Benchmark studies<a class="anchor-link" href="#benchm
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0b549abb">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=44c6462b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5439,7 +5476,7 @@ <h1 id="benchmark-studies">Benchmark studies<a class="anchor-link" href="#benchm
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3e8f6c0a">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2d864094">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5454,7 +5491,7 @@ <h2 id="listing-studies">Listing studies<a class="anchor-link" href="#listing-st
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e2511c08">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2b8efdfd">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5483,7 +5520,7 @@ <h2 id="listing-studies">Listing studies<a class="anchor-link" href="#listing-st
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8478ddd2">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=02f422dc">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5494,7 +5531,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2ae70e12">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ffbcb23f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5505,7 +5542,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b5274094">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d8434967">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5534,7 +5571,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=48876f59">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=62024710">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5545,7 +5582,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0d21f41e">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=57dddfa6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5572,7 +5609,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=412c6e3d">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=322d155d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5583,7 +5620,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=97a0985c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c4be4aec">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5610,7 +5647,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a5b3fe84">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=558439b3">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5622,7 +5659,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c7b84373">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=93218e4d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5659,7 +5696,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=225564f8">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=716bec8b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5672,7 +5709,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2be3b1c3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=87413a15">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5699,7 +5736,7 @@ <h2 id="downloading-studies">Downloading studies<a class="anchor-link" href="#do
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f2edaff3">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0150aabc">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5712,7 +5749,7 @@ <h2 id="uploading-studies">Uploading studies<a class="anchor-link" href="#upload
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e58b91a9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6d33c42d">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5797,7 +5834,7 @@ <h2 id="uploading-studies">Uploading studies<a class="anchor-link" href="#upload
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=cc5bf8d9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0e002886">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/suites_tutorial/index.html b/examples/30_extended/suites_tutorial/index.html
index 07652d9..4794525 100644
--- a/examples/30_extended/suites_tutorial/index.html
+++ b/examples/30_extended/suites_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4536,6 +4553,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4553,7 +4590,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4569,14 +4606,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4642,7 +4679,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4658,14 +4695,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4721,7 +4758,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4737,14 +4774,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5389,7 +5426,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ff082fd0">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a8ed5d0a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5403,7 +5440,7 @@ <h1 id="benchmark-suites">Benchmark suites<a class="anchor-link" href="#benchmar
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4eba8adb">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b25a334a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5438,7 +5475,7 @@ <h1 id="benchmark-suites">Benchmark suites<a class="anchor-link" href="#benchmar
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4625b00a">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8134f603">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5453,7 +5490,7 @@ <h2 id="listing-suites">Listing suites<a class="anchor-link" href="#listing-suit
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7e9ad7c1">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d39424b7">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5482,7 +5519,7 @@ <h2 id="listing-suites">Listing suites<a class="anchor-link" href="#listing-suit
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8fac1f95">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4ce01df0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5493,7 +5530,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8fec710d">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=add4cf60">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5504,7 +5541,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=62d447ad">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e7c885a4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5533,7 +5570,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b53aad8c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=63777ab4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5544,7 +5581,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5369e1aa">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f82b8c01">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5571,7 +5608,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=68167c60">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b9998855">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5582,7 +5619,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=82dbb2d6">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=34dfee22">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5609,7 +5646,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=86833dbc">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6a473a75">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5620,7 +5657,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=183f7d54">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=caf3abb5">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5647,7 +5684,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ba464d4b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f39e3c33">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5660,7 +5697,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=326c43ec">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b5d9e645">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5689,7 +5726,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0e65e68c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=76e85eb0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5702,7 +5739,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5a6ac2bf">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=227f0cab">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5729,7 +5766,7 @@ <h2 id="downloading-suites">Downloading suites<a class="anchor-link" href="#down
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=eee65ca2">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=14c4f6e1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5744,7 +5781,7 @@ <h2 id="uploading-suites">Uploading suites<a class="anchor-link" href="#uploadin
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f4c2c230">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=70483f9f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5801,7 +5838,7 @@ <h2 id="uploading-suites">Uploading suites<a class="anchor-link" href="#uploadin
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=fb3425f9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c70b6d87">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/task_manual_iteration_tutorial/index.html b/examples/30_extended/task_manual_iteration_tutorial/index.html
index a748f2b..e698093 100644
--- a/examples/30_extended/task_manual_iteration_tutorial/index.html
+++ b/examples/30_extended/task_manual_iteration_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4479,6 +4496,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4496,7 +4533,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4512,14 +4549,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4664,7 +4701,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4680,14 +4717,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5297,7 +5334,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6d170296">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e3563c94">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5313,7 +5350,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0f4b22db">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=872c3800">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5340,7 +5377,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1ee9a58c">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=14f51389">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5356,7 +5393,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=57074641">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ab28fbc1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5385,7 +5422,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=657a97fd">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=dc5a1e29">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5397,7 +5434,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7dc26186">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3ecd5938">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5424,7 +5461,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=722b5974">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1b4f06b8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5443,7 +5480,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f2666042">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=549a2bae">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5484,7 +5521,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d5433a50">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=58aa8d88">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5497,7 +5534,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=fb47f19c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=eaad0e83">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5538,7 +5575,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=909ad57e">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=653b5c8f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5549,7 +5586,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8f6a0119">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=daa7b384">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5602,7 +5639,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=44cb73a5">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=56f3fd2a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5613,7 +5650,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d93aaeef">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8024c64f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5662,7 +5699,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2a7aac01">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4c44bffa">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5673,7 +5710,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0396bd01">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=64ea9193">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5748,7 +5785,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=05b57070">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=20e4fc5b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5759,7 +5796,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3a2dca79">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=57ab9c03">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5808,7 +5845,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e04d8c87">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=57f65433">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5819,7 +5856,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=82d6ac24">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f2ac2c4b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5894,7 +5931,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=e50e4806">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5bb9d89a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5905,7 +5942,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c18e595c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=10692bd4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5954,7 +5991,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a0467e9b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6ce6cbcb">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5965,7 +6002,7 @@ <h1 id="tasks-retrieving-splits">Tasks: retrieving splits<a class="anchor-link"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f691be70">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=1727f4a6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/30_extended/tasks_tutorial/index.html b/examples/30_extended/tasks_tutorial/index.html
index c656efb..04a422f 100644
--- a/examples/30_extended/tasks_tutorial/index.html
+++ b/examples/30_extended/tasks_tutorial/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4554,6 +4571,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4660,7 +4697,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4676,14 +4713,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4739,7 +4776,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4755,14 +4792,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5425,7 +5462,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=58b9d5d2">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=19d13408">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5436,7 +5473,7 @@ <h1 id="tasks">Tasks<a class="anchor-link" href="#tasks">¶</a></h1><p>A tutoria
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ad585c43">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4010865a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5467,7 +5504,7 @@ <h1 id="tasks">Tasks<a class="anchor-link" href="#tasks">¶</a></h1><p>A tutoria
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7a730ffa">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f6b7a5c6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5489,7 +5526,7 @@ <h1 id="tasks">Tasks<a class="anchor-link" href="#tasks">¶</a></h1><p>A tutoria
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6891b326">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=eecc9a20">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5504,7 +5541,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=be670e4c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7c9ae367">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5541,7 +5578,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6053e682">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=1fd70db8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5553,7 +5590,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c7b5ce25">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6fd56a0a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5582,7 +5619,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=4635ddc5">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9a816d3e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5611,7 +5648,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ef554ee2">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=73388382">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5622,7 +5659,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e7163fc9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=49563494">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5651,7 +5688,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8a503319">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=91ba2d8a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5680,7 +5717,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c37b4263">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=165b7055">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5693,7 +5730,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c515a339">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=880d0b07">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5724,7 +5761,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ff2753de">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=3f3ffe3b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5735,7 +5772,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=dacfc7a7">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e54fa4ea">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5766,7 +5803,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5b7d6b15">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=940a5a92">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5777,7 +5814,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0c8cb4af">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=55012538">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5806,7 +5843,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8c736193">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f31e2afe">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5824,7 +5861,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=77506307">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d200252b">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5853,7 +5890,7 @@ <h2 id="listing-tasks">Listing tasks<a class="anchor-link" href="#listing-tasks"
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=df5b7937">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c3d74e8c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5864,7 +5901,7 @@ <h2 id="exercise">Exercise<a class="anchor-link" href="#exercise">¶</a></h2><p>
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=fba592c5">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=14548022">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5891,7 +5928,7 @@ <h2 id="exercise">Exercise<a class="anchor-link" href="#exercise">¶</a></h2><p>
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=89e28309">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=191fe1e4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5904,7 +5941,7 @@ <h2 id="downloading-tasks">Downloading tasks<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=0f5c66cd">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f32ac8ed">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5933,7 +5970,7 @@ <h2 id="downloading-tasks">Downloading tasks<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=792d77ee">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c551bacf">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5962,7 +5999,7 @@ <h2 id="downloading-tasks">Downloading tasks<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f88e27a7">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=fec80936">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5997,7 +6034,7 @@ <h2 id="downloading-tasks">Downloading tasks<a class="anchor-link" href="#downlo
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d88ecb24">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=0d4f8c1c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6030,7 +6067,7 @@ <h2 id="creating-tasks">Creating tasks<a class="anchor-link" href="#creating-tas
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=80165edd">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=f2c499d0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6043,7 +6080,7 @@ <h2 id="creating-tasks">Creating tasks<a class="anchor-link" href="#creating-tas
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b75fe791">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=ba6de08a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6070,7 +6107,7 @@ <h2 id="creating-tasks">Creating tasks<a class="anchor-link" href="#creating-tas
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7a4e2525">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=55a396bc">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6086,7 +6123,7 @@ <h2 id="example">Example<a class="anchor-link" href="#example">¶</a></h2><p>Let
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7012fa6b">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e173923f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6153,7 +6190,7 @@ <h2 id="example">Example<a class="anchor-link" href="#example">¶</a></h2><p>Let
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=062705b3">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=71fd043f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6182,7 +6219,7 @@ <h2 id="example">Example<a class="anchor-link" href="#example">¶</a></h2><p>Let
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=58819bcc">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=97170697">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -6239,29 +6276,6 @@ <h2 id="example">Example<a class="anchor-link" href="#example">¶</a></h2><p>Let
     
     
     
-      
-  <span class="md-source-file__fact">
-    
-      
-  <span class="md-icon" title="Contributors">
-    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2"/></svg>
-  </span>
-  <span>GitHub</span>
-
-    
-    <nav>
-      
-        <a href="https://github.com/joaquinvanschoren" class="md-author" title="@joaquinvanschoren">
-          
-          <img src="https://avatars.githubusercontent.com/u/1724503?v=4&size=72" alt="joaquinvanschoren">
-        </a>
-      
-      
-      
-    </nav>
-  </span>
-
-    
   </aside>
 
 
diff --git a/examples/40_paper/2015_neurips_feurer_example/index.html b/examples/40_paper/2015_neurips_feurer_example/index.html
index 43608ca..53dcfe7 100644
--- a/examples/40_paper/2015_neurips_feurer_example/index.html
+++ b/examples/40_paper/2015_neurips_feurer_example/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4518,6 +4535,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4535,7 +4572,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4551,14 +4588,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4624,7 +4661,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4640,14 +4677,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4703,7 +4740,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4719,14 +4756,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5353,7 +5390,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a55f3d64">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8f3442d8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5369,7 +5406,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=cdae2a85">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=cc87a3cf">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5400,7 +5437,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8ae21879">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=7bd5836f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5412,7 +5449,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d094c6b8">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=39e3cd7f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5461,7 +5498,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=98accf50">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=912d0b1f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5481,7 +5518,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6ac640da">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=fc578fb0">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5495,7 +5532,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=78d6b696">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=28b5362a">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/40_paper/2018_ida_strang_example/index.html b/examples/40_paper/2018_ida_strang_example/index.html
index e5c1e07..002e36b 100644
--- a/examples/40_paper/2018_ida_strang_example/index.html
+++ b/examples/40_paper/2018_ida_strang_example/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4518,6 +4535,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4535,7 +4572,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4551,14 +4588,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4624,7 +4661,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4640,14 +4677,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4703,7 +4740,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4719,14 +4756,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5353,7 +5390,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=59825b95">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6cf7eb9f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5369,7 +5406,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=35771cd9">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=3e744315">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5400,7 +5437,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4e160fa1">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=12b7cdf9">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5422,7 +5459,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d09b8305">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d77d45df">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5505,7 +5542,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6d69cfb5">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=fbeabeca">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5516,7 +5553,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f341503a">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b32e6432">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5555,7 +5592,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=b460b3c2">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d9870d99">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5567,7 +5604,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=90755204">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e353d02f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5642,7 +5679,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6d84b21b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2da03ae8">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5654,7 +5691,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f2b11bf4">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8e6281d4">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/40_paper/2018_kdd_rijn_example/index.html b/examples/40_paper/2018_kdd_rijn_example/index.html
index 7cacb87..b3ef29f 100644
--- a/examples/40_paper/2018_kdd_rijn_example/index.html
+++ b/examples/40_paper/2018_kdd_rijn_example/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4518,6 +4535,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4535,7 +4572,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4551,14 +4588,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4624,7 +4661,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4640,14 +4677,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4703,7 +4740,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4719,14 +4756,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5353,7 +5390,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=bdb9995b">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=43c6db1e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5382,7 +5419,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ce39fba7">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5de2a6f1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5420,7 +5457,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2629c97c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=6b982c5e">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5627,7 +5664,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=9133aac5">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=5948e886">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5641,7 +5678,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9a0520bb">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7a53c502">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/40_paper/2018_neurips_perrone_example/index.html b/examples/40_paper/2018_neurips_perrone_example/index.html
index 65e84d8..131e1eb 100644
--- a/examples/40_paper/2018_neurips_perrone_example/index.html
+++ b/examples/40_paper/2018_neurips_perrone_example/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4545,6 +4562,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4562,7 +4599,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4578,14 +4615,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4651,7 +4688,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4667,14 +4704,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4730,7 +4767,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4746,14 +4783,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5407,7 +5444,7 @@
 </style>
 <!-- End of mermaid configuration --><div class="jupyter-wrapper">
 <div class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8060e4b5">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=6f303a01">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5431,7 +5468,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f98d23cc">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=39e84cd2">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5480,7 +5517,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=4e11b46f">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=d879f2f3">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5492,7 +5529,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=9984a0ab">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=cc919502">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5733,7 +5770,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=ebf716a9">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=a1e11175">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5748,7 +5785,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=fdc25401">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d8c4c27c">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5781,7 +5818,7 @@ <h2 id="publication">Publication<a class="anchor-link" href="#publication">¶</a
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=dde95bf3">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=dd774b53">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5794,7 +5831,7 @@ <h2 id="creating-pre-processing-and-modelling-pipelines">Creating pre-processing
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=5666ae60">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d3b0abac">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5851,7 +5888,7 @@ <h2 id="creating-pre-processing-and-modelling-pipelines">Creating pre-processing
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=8e9b49d4">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=241f089f">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5864,7 +5901,7 @@ <h2 id="building-a-surrogate-model-on-a-tasks-evaluation">Building a surrogate m
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=97bdef7d">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=63165d16">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5907,7 +5944,7 @@ <h2 id="building-a-surrogate-model-on-a-tasks-evaluation">Building a surrogate m
 </div>
 </div>
 </div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=74d0d3d0">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c9fb8ee6">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5921,7 +5958,7 @@ <h2 id="evaluating-the-surrogate-model">Evaluating the surrogate model<a class="
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=d01a055c">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=91d69cd1">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
@@ -5988,7 +6025,7 @@ <h2 id="evaluating-the-surrogate-model">Evaluating the surrogate model<a class="
 </div>
 </div>
 </div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8833e706">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=b0b798bb">
 <div class="jp-Cell-inputWrapper" tabindex="0">
 <div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
 </div>
diff --git a/examples/SUMMARY/index.html b/examples/SUMMARY/index.html
index a72b9cb..650d309 100644
--- a/examples/SUMMARY/index.html
+++ b/examples/SUMMARY/index.html
@@ -362,6 +362,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4449,6 +4466,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4466,7 +4503,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4482,14 +4519,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4555,7 +4592,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4571,14 +4608,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4634,7 +4671,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4650,14 +4687,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/help/index.html b/help/index.html
index b4c7e17..ed90ca8 100644
--- a/help/index.html
+++ b/help/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4549,6 +4566,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4566,7 +4603,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4582,14 +4619,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4655,7 +4692,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4671,14 +4708,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4734,7 +4771,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4750,14 +4787,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/index.html b/index.html
index 73bd8d2..55713d2 100644
--- a/index.html
+++ b/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4463,6 +4480,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4480,7 +4517,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4496,14 +4533,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4569,7 +4606,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4585,14 +4622,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4648,7 +4685,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4664,14 +4701,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5001,6 +5038,44 @@ <h2 id="want-to-get-involved">Want to get involved?<a class="headerlink" href="#
     
     
     
+      
+  <span class="md-source-file__fact">
+    
+      
+  <span class="md-icon" title="Contributors">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2"/></svg>
+  </span>
+  <span>GitHub</span>
+
+    
+    <nav>
+      
+        <a href="https://github.com/joaquinvanschoren" class="md-author" title="@joaquinvanschoren">
+          
+          <img src="https://avatars.githubusercontent.com/u/1724503?v=4&size=72" alt="joaquinvanschoren">
+        </a>
+      
+        <a href="https://github.com/SubhadityaMukherjee" class="md-author" title="@SubhadityaMukherjee">
+          
+          <img src="https://avatars.githubusercontent.com/u/26865436?v=4&size=72" alt="SubhadityaMukherjee">
+        </a>
+      
+        <a href="https://github.com/PGijsbers" class="md-author" title="@PGijsbers">
+          
+          <img src="https://avatars.githubusercontent.com/u/15890747?v=4&size=72" alt="PGijsbers">
+        </a>
+      
+        <a href="https://github.com/janvanrijn" class="md-author" title="@janvanrijn">
+          
+          <img src="https://avatars.githubusercontent.com/u/2292730?v=4&size=72" alt="janvanrijn">
+        </a>
+      
+      
+      
+    </nav>
+  </span>
+
+    
   </aside>
 
 
diff --git a/intro/Governance/index.html b/intro/Governance/index.html
index 19a2882..59b7d8b 100644
--- a/intro/Governance/index.html
+++ b/intro/Governance/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4561,6 +4578,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4578,7 +4615,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4594,14 +4631,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4667,7 +4704,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4683,14 +4720,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4746,7 +4783,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4762,14 +4799,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/intro/showcase/index.html b/intro/showcase/index.html
index 9677bdc..7142570 100644
--- a/intro/showcase/index.html
+++ b/intro/showcase/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4475,6 +4492,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4492,7 +4529,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4508,14 +4545,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4581,7 +4618,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4597,14 +4634,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4660,7 +4697,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4676,14 +4713,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/intro/terms/index.html b/intro/terms/index.html
index f1a8939..ca2981e 100644
--- a/intro/terms/index.html
+++ b/intro/terms/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4519,6 +4536,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4536,7 +4573,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4552,14 +4589,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4625,7 +4662,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4641,14 +4678,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4704,7 +4741,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4720,14 +4757,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/julia/index.html b/julia/index.html
index 0ca926f..6a6ac40 100644
--- a/julia/index.html
+++ b/julia/index.html
@@ -11,7 +11,7 @@
         <link rel="canonical" href="https://docs.openml.org/julia/">
       
       
-        <link rel="prev" href="../pytorch/Philosophy%20behind%20the%20API%20Design/">
+        <link rel="prev" href="../tensorflow/">
       
       
         <link rel="next" href="../r/">
@@ -370,6 +370,23 @@
         
   
   
+  
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
     
   
   
@@ -4457,6 +4474,26 @@
       
       
   
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
   
     
   
@@ -4482,7 +4519,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" checked>
         
           
           
@@ -4498,14 +4535,14 @@
             </a>
             
               
-              <label class="md-nav__link md-nav__link--active" for="__nav_8" id="__nav_8_label" tabindex="">
+              <label class="md-nav__link md-nav__link--active" for="__nav_9" id="__nav_9_label" tabindex="">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4650,7 +4687,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4666,14 +4703,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5444,7 +5481,7 @@ <h3 id="openmlload"><a id='OpenML.load' href='#OpenML.load'><strong><code>OpenML
       <nav class="md-footer__inner md-grid" aria-label="Footer" >
         
           
-          <a href="../pytorch/Philosophy%20behind%20the%20API%20Design/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Philosphy behind the API Design">
+          <a href="../tensorflow/" class="md-footer__link md-footer__link--prev" aria-label="Previous: TensorFlow">
             <div class="md-footer__button md-icon">
               
               <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
@@ -5454,7 +5491,7 @@ <h3 id="openmlload"><a id='OpenML.load' href='#OpenML.load'><strong><code>OpenML
                 Previous
               </span>
               <div class="md-ellipsis">
-                Philosphy behind the API Design
+                TensorFlow
               </div>
             </div>
           </a>
diff --git a/notebooks/getting_started/index.html b/notebooks/getting_started/index.html
index 1b1bcc7..1a48d47 100644
--- a/notebooks/getting_started/index.html
+++ b/notebooks/getting_started/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4532,6 +4549,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4549,7 +4586,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4565,14 +4602,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4638,7 +4675,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4654,14 +4691,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4717,7 +4754,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4733,14 +4770,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/python/CONTRIBUTING/index.html b/python/CONTRIBUTING/index.html
index 08e9605..7a81b56 100644
--- a/python/CONTRIBUTING/index.html
+++ b/python/CONTRIBUTING/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5191,7 +5228,7 @@ <h2 id="documentation">Documentation<a class="headerlink" href="#documentation"
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5201,7 +5238,7 @@ <h2 id="documentation">Documentation<a class="headerlink" href="#documentation"
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/ISSUE_TEMPLATE/index.html b/python/ISSUE_TEMPLATE/index.html
index dd0cf77..9613190 100644
--- a/python/ISSUE_TEMPLATE/index.html
+++ b/python/ISSUE_TEMPLATE/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -4944,7 +4981,7 @@ <h4 id="versions">Versions<a class="headerlink" href="#versions" title="Permanen
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -4954,7 +4991,7 @@ <h4 id="versions">Versions<a class="headerlink" href="#versions" title="Permanen
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/PULL_REQUEST_TEMPLATE/index.html b/python/PULL_REQUEST_TEMPLATE/index.html
index 45a0894..eb1abda 100644
--- a/python/PULL_REQUEST_TEMPLATE/index.html
+++ b/python/PULL_REQUEST_TEMPLATE/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -4925,7 +4962,7 @@ <h4 id="any-other-comments">Any other comments?<a class="headerlink" href="#any-
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -4935,7 +4972,7 @@ <h4 id="any-other-comments">Any other comments?<a class="headerlink" href="#any-
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/contributing/index.html b/python/contributing/index.html
index bc5e23f..b177c2c 100644
--- a/python/contributing/index.html
+++ b/python/contributing/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4475,6 +4492,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4492,7 +4529,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4508,14 +4545,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4581,7 +4618,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4597,14 +4634,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4660,7 +4697,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4676,14 +4713,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/python/docker/readme/index.html b/python/docker/readme/index.html
index 56dd8ae..f4236a8 100644
--- a/python/docker/readme/index.html
+++ b/python/docker/readme/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5067,7 +5104,7 @@ <h2 id="building-the-image">Building the image<a class="headerlink" href="#build
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5077,7 +5114,7 @@ <h2 id="building-the-image">Building the image<a class="headerlink" href="#build
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/docs/contributing/index.html b/python/docs/contributing/index.html
index 26cc4ee..d0bdfb4 100644
--- a/python/docs/contributing/index.html
+++ b/python/docs/contributing/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -4883,7 +4920,7 @@ <h1 id="contributing">Contributing<a class="headerlink" href="#contributing" tit
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -4893,7 +4930,7 @@ <h1 id="contributing">Contributing<a class="headerlink" href="#contributing" tit
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/docs/extensions/index.html b/python/docs/extensions/index.html
index 400d6f3..7380ad8 100644
--- a/python/docs/extensions/index.html
+++ b/python/docs/extensions/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5118,7 +5155,7 @@ <h3 id="recommended">Recommended<a class="headerlink" href="#recommended" title=
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5128,7 +5165,7 @@ <h3 id="recommended">Recommended<a class="headerlink" href="#recommended" title=
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/docs/index.html b/python/docs/index.html
index c125b19..d7a6cbb 100644
--- a/python/docs/index.html
+++ b/python/docs/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5013,7 +5050,7 @@ <h2 id="citing-openml-python">Citing OpenML-Python<a class="headerlink" href="#c
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5023,7 +5060,7 @@ <h2 id="citing-openml-python">Citing OpenML-Python<a class="headerlink" href="#c
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/docs/progress/index.html b/python/docs/progress/index.html
index c501379..4eb5d04 100644
--- a/python/docs/progress/index.html
+++ b/python/docs/progress/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5520,7 +5557,7 @@ <h2 id="prior">Prior<a class="headerlink" href="#prior" title="Permanent link">&
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5530,7 +5567,7 @@ <h2 id="prior">Prior<a class="headerlink" href="#prior" title="Permanent link">&
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/docs/usage/index.html b/python/docs/usage/index.html
index f1a69a7..8b426aa 100644
--- a/python/docs/usage/index.html
+++ b/python/docs/usage/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5058,7 +5095,7 @@ <h1 id="extending-openml-python">Extending OpenML-Python<a class="headerlink" hr
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5068,7 +5105,7 @@ <h1 id="extending-openml-python">Extending OpenML-Python<a class="headerlink" hr
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/python/extensions/index.html b/python/extensions/index.html
index 7c54f09..daca163 100644
--- a/python/extensions/index.html
+++ b/python/extensions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4580,6 +4597,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4597,7 +4634,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4613,14 +4650,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4686,7 +4723,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4702,14 +4739,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4765,7 +4802,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4781,14 +4818,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/python/index.html b/python/index.html
index acef147..359fb67 100644
--- a/python/index.html
+++ b/python/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4465,6 +4482,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4482,7 +4519,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4498,14 +4535,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4650,7 +4687,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4666,14 +4703,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/python/progress/index.html b/python/progress/index.html
index 6835c84..7d92b79 100644
--- a/python/progress/index.html
+++ b/python/progress/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4667,6 +4684,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4684,7 +4721,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4700,14 +4737,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4773,7 +4810,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4789,14 +4826,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4852,7 +4889,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4868,14 +4905,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/python/usage/index.html b/python/usage/index.html
index 832756d..b6401dd 100644
--- a/python/usage/index.html
+++ b/python/usage/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4568,6 +4585,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4674,7 +4711,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4690,14 +4727,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4753,7 +4790,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4769,14 +4806,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/API reference/Callbacks/index.html b/pytorch/API reference/Callbacks/index.html
index b615093..156bd01 100644
--- a/pytorch/API reference/Callbacks/index.html	
+++ b/pytorch/API reference/Callbacks/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4726,6 +4743,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4743,7 +4780,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4759,14 +4796,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4832,7 +4869,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4848,14 +4885,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4911,7 +4948,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4927,14 +4964,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/API reference/Custom Datasets/index.html b/pytorch/API reference/Custom Datasets/index.html
index 04cacab..ae401e5 100644
--- a/pytorch/API reference/Custom Datasets/index.html	
+++ b/pytorch/API reference/Custom Datasets/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4534,6 +4551,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4551,7 +4588,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4567,14 +4604,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4640,7 +4677,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4656,14 +4693,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4719,7 +4756,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4735,14 +4772,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/API reference/Metrics/index.html b/pytorch/API reference/Metrics/index.html
index b9aca7f..d5d7169 100644
--- a/pytorch/API reference/Metrics/index.html	
+++ b/pytorch/API reference/Metrics/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4543,6 +4560,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4649,7 +4686,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4665,14 +4702,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4728,7 +4765,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4744,14 +4781,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/API reference/OpenML Connection/index.html b/pytorch/API reference/OpenML Connection/index.html
index 0f4340a..fdfa17f 100644
--- a/pytorch/API reference/OpenML Connection/index.html	
+++ b/pytorch/API reference/OpenML Connection/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4639,6 +4656,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4656,7 +4693,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4672,14 +4709,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4745,7 +4782,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4761,14 +4798,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4824,7 +4861,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4840,14 +4877,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/API reference/Trainer/index.html b/pytorch/API reference/Trainer/index.html
index ae29ffc..4ca54b1 100644
--- a/pytorch/API reference/Trainer/index.html	
+++ b/pytorch/API reference/Trainer/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4612,6 +4629,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4629,7 +4666,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4645,14 +4682,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4718,7 +4755,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4734,14 +4771,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4797,7 +4834,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4813,14 +4850,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/Examples/Create Dataset and Task/index.html b/pytorch/Examples/Create Dataset and Task/index.html
index 72e06de..d393774 100644
--- a/pytorch/Examples/Create Dataset and Task/index.html	
+++ b/pytorch/Examples/Create Dataset and Task/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4477,6 +4494,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4494,7 +4531,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4510,14 +4547,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4583,7 +4620,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4599,14 +4636,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4662,7 +4699,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4678,14 +4715,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/Examples/Image Classification Task/index.html b/pytorch/Examples/Image Classification Task/index.html
index babfac9..e26bfe4 100644
--- a/pytorch/Examples/Image Classification Task/index.html	
+++ b/pytorch/Examples/Image Classification Task/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4588,6 +4605,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4605,7 +4642,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4621,14 +4658,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4694,7 +4731,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4710,14 +4747,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4773,7 +4810,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4789,14 +4826,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/Examples/Pretrained Transformer Image Classification Task/index.html b/pytorch/Examples/Pretrained Transformer Image Classification Task/index.html
index ceb0ce8..d86caa1 100644
--- a/pytorch/Examples/Pretrained Transformer Image Classification Task/index.html	
+++ b/pytorch/Examples/Pretrained Transformer Image Classification Task/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4579,6 +4596,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4596,7 +4633,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4612,14 +4649,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4685,7 +4722,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4701,14 +4738,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4764,7 +4801,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4780,14 +4817,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/Examples/Sequential Classification Task/index.html b/pytorch/Examples/Sequential Classification Task/index.html
index 7ffe45a..f713fc4 100644
--- a/pytorch/Examples/Sequential Classification Task/index.html	
+++ b/pytorch/Examples/Sequential Classification Task/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4579,6 +4596,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4596,7 +4633,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4612,14 +4649,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4685,7 +4722,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4701,14 +4738,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4764,7 +4801,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4780,14 +4817,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/Examples/Tabular Classification/index.html b/pytorch/Examples/Tabular Classification/index.html
index 7d2e2c3..d720526 100644
--- a/pytorch/Examples/Tabular Classification/index.html	
+++ b/pytorch/Examples/Tabular Classification/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4477,6 +4494,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4494,7 +4531,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4510,14 +4547,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4583,7 +4620,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4599,14 +4636,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4662,7 +4699,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4678,14 +4715,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/Limitations of the API/index.html b/pytorch/Limitations of the API/index.html
index e1e5a3d..c4fa448 100644
--- a/pytorch/Limitations of the API/index.html	
+++ b/pytorch/Limitations of the API/index.html	
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4475,6 +4492,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4492,7 +4529,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4508,14 +4545,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4581,7 +4618,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4597,14 +4634,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4660,7 +4697,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4676,14 +4713,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/pytorch/Philosophy behind the API Design/index.html b/pytorch/Philosophy behind the API Design/index.html
index da42e4f..48d72d9 100644
--- a/pytorch/Philosophy behind the API Design/index.html	
+++ b/pytorch/Philosophy behind the API Design/index.html	
@@ -14,7 +14,7 @@
         <link rel="prev" href="../Limitations%20of%20the%20API/">
       
       
-        <link rel="next" href="../../julia/">
+        <link rel="next" href="../../tensorflow/">
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4514,6 +4531,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4531,7 +4568,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4547,14 +4584,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4620,7 +4657,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4636,14 +4673,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4699,7 +4736,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4715,14 +4752,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5000,13 +5037,13 @@ <h2 id="separation-of-concerns">Separation of Concerns<a class="headerlink" href
         
         
           
-          <a href="../../julia/" class="md-footer__link md-footer__link--next" aria-label="Next: Home">
+          <a href="../../tensorflow/" class="md-footer__link md-footer__link--next" aria-label="Next: TensorFlow">
             <div class="md-footer__title">
               <span class="md-footer__direction">
                 Next
               </span>
               <div class="md-ellipsis">
-                Home
+                TensorFlow
               </div>
             </div>
             <div class="md-footer__button md-icon">
diff --git a/pytorch/index.html b/pytorch/index.html
index 907eb75..fa889e9 100644
--- a/pytorch/index.html
+++ b/pytorch/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4465,6 +4482,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4482,7 +4519,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4498,14 +4535,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4571,7 +4608,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4587,14 +4624,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4650,7 +4687,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4666,14 +4703,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
diff --git a/r/index.html b/r/index.html
index d8e64b9..8f4736e 100644
--- a/r/index.html
+++ b/r/index.html
@@ -371,6 +371,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4460,6 +4477,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4477,7 +4514,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4493,14 +4530,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4642,7 +4679,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4658,14 +4695,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4721,7 +4758,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4737,14 +4774,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5154,7 +5191,7 @@ <h2 id="bugs-questions-feedback">Bugs, Questions, Feedback<a class="headerlink"
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5164,7 +5201,7 @@ <h2 id="bugs-questions-feedback">Bugs, Questions, Feedback<a class="headerlink"
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/SUMMARY/index.html b/reference/SUMMARY/index.html
index 6d20143..9810cb6 100644
--- a/reference/SUMMARY/index.html
+++ b/reference/SUMMARY/index.html
@@ -362,6 +362,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4449,6 +4466,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4466,7 +4503,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4482,14 +4519,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4555,7 +4592,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4571,14 +4608,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4634,7 +4671,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4650,14 +4687,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -4893,7 +4930,7 @@ <h1>SUMMARY</h1>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -4903,7 +4940,7 @@ <h1>SUMMARY</h1>
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/__version__/index.html b/reference/__version__/index.html
index 1a15c6f..b87a68e 100644
--- a/reference/__version__/index.html
+++ b/reference/__version__/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -4909,7 +4946,7 @@ <h1>  version  </h1>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -4919,7 +4956,7 @@ <h1>  version  </h1>
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/_api_calls/index.html b/reference/_api_calls/index.html
index 67d54f1..3163c97 100644
--- a/reference/_api_calls/index.html
+++ b/reference/_api_calls/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5028,7 +5065,7 @@ <h2 id="openml._api_calls.resolve_env_proxies" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5038,7 +5075,7 @@ <h2 id="openml._api_calls.resolve_env_proxies" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/base/index.html b/reference/base/index.html
index 24c3ff4..18b1e1f 100644
--- a/reference/base/index.html
+++ b/reference/base/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4590,6 +4607,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4607,7 +4644,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4623,14 +4660,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4696,7 +4733,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4712,14 +4749,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4775,7 +4812,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4791,14 +4828,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5785,7 +5822,7 @@ <h3 id="openml.base.OpenMLBase.url_for_id" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5795,7 +5832,7 @@ <h3 id="openml.base.OpenMLBase.url_for_id" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/cli/index.html b/reference/cli/index.html
index 154b60e..4a396fc 100644
--- a/reference/cli/index.html
+++ b/reference/cli/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4539,6 +4556,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4556,7 +4593,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4572,14 +4609,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4645,7 +4682,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4661,14 +4698,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4724,7 +4761,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4740,14 +4777,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5499,7 +5536,7 @@ <h2 id="openml.cli.wait_until_valid_input" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5509,7 +5546,7 @@ <h2 id="openml.cli.wait_until_valid_input" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/config/index.html b/reference/config/index.html
index 13e3d6d..400b84a 100644
--- a/reference/config/index.html
+++ b/reference/config/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4614,6 +4631,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4631,7 +4668,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4647,14 +4684,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4720,7 +4757,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4736,14 +4773,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4799,7 +4836,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4815,14 +4852,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5829,7 +5866,7 @@ <h2 id="openml.config.set_root_cache_directory" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5839,7 +5876,7 @@ <h2 id="openml.config.set_root_cache_directory" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/datasets/data_feature/index.html b/reference/datasets/data_feature/index.html
index 537be5c..ecfd300 100644
--- a/reference/datasets/data_feature/index.html
+++ b/reference/datasets/data_feature/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4467,6 +4484,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4484,7 +4521,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4500,14 +4537,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4573,7 +4610,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4589,14 +4626,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5237,7 +5274,7 @@ <h2 id="openml.datasets.data_feature.OpenMLDataFeature" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5247,7 +5284,7 @@ <h2 id="openml.datasets.data_feature.OpenMLDataFeature" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/datasets/dataset/index.html b/reference/datasets/dataset/index.html
index 3b70a57..8f32b66 100644
--- a/reference/datasets/dataset/index.html
+++ b/reference/datasets/dataset/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4467,6 +4484,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4484,7 +4521,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4500,14 +4537,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4573,7 +4610,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4589,14 +4626,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -8426,7 +8463,7 @@ <h3 id="openml.datasets.dataset.OpenMLDataset.retrieve_class_labels" class="doc
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -8436,7 +8473,7 @@ <h3 id="openml.datasets.dataset.OpenMLDataset.retrieve_class_labels" class="doc
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/datasets/functions/index.html b/reference/datasets/functions/index.html
index 2edbf8a..f7f40c3 100644
--- a/reference/datasets/functions/index.html
+++ b/reference/datasets/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4467,6 +4484,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4484,7 +4521,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4500,14 +4537,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4573,7 +4610,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4589,14 +4626,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -8554,7 +8591,7 @@ <h2 id="openml.datasets.functions.status_update" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -8564,7 +8601,7 @@ <h2 id="openml.datasets.functions.status_update" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/datasets/index.html b/reference/datasets/index.html
index 78c614c..8c41731 100644
--- a/reference/datasets/index.html
+++ b/reference/datasets/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4467,6 +4484,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4484,7 +4521,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4500,14 +4537,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4573,7 +4610,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4589,14 +4626,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -12042,7 +12079,7 @@ <h2 id="openml.datasets.status_update" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -12052,7 +12089,7 @@ <h2 id="openml.datasets.status_update" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/evaluations/evaluation/index.html b/reference/evaluations/evaluation/index.html
index 2f6583e..9e6b4b4 100644
--- a/reference/evaluations/evaluation/index.html
+++ b/reference/evaluations/evaluation/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4523,6 +4540,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4540,7 +4577,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4556,14 +4593,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4629,7 +4666,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4645,14 +4682,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4708,7 +4745,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4724,14 +4761,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5491,7 +5528,7 @@ <h2 id="openml.evaluations.evaluation.OpenMLEvaluation" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5501,7 +5538,7 @@ <h2 id="openml.evaluations.evaluation.OpenMLEvaluation" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/evaluations/functions/index.html b/reference/evaluations/functions/index.html
index 4b702ca..10aaa35 100644
--- a/reference/evaluations/functions/index.html
+++ b/reference/evaluations/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4559,6 +4576,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4576,7 +4613,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4592,14 +4629,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4665,7 +4702,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4681,14 +4718,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4744,7 +4781,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4760,14 +4797,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -6388,7 +6425,7 @@ <h2 id="openml.evaluations.functions.list_evaluations_setups" class="doc doc-hea
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -6398,7 +6435,7 @@ <h2 id="openml.evaluations.functions.list_evaluations_setups" class="doc doc-hea
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/evaluations/index.html b/reference/evaluations/index.html
index 6173222..dccf801 100644
--- a/reference/evaluations/index.html
+++ b/reference/evaluations/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4469,6 +4486,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4486,7 +4523,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4502,14 +4539,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4575,7 +4612,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4591,14 +4628,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4654,7 +4691,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4670,14 +4707,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -6514,7 +6551,7 @@ <h2 id="openml.evaluations.list_evaluations_setups" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -6524,7 +6561,7 @@ <h2 id="openml.evaluations.list_evaluations_setups" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/exceptions/index.html b/reference/exceptions/index.html
index 8bc3ece..0525509 100644
--- a/reference/exceptions/index.html
+++ b/reference/exceptions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4602,6 +4619,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4619,7 +4656,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4635,14 +4672,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4708,7 +4745,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4724,14 +4761,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4787,7 +4824,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4803,14 +4840,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5707,7 +5744,7 @@ <h2 id="openml.exceptions.PyOpenMLError" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5717,7 +5754,7 @@ <h2 id="openml.exceptions.PyOpenMLError" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/extensions/extension_interface/index.html b/reference/extensions/extension_interface/index.html
index 42bfdd5..65c9c09 100644
--- a/reference/extensions/extension_interface/index.html
+++ b/reference/extensions/extension_interface/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4628,6 +4645,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4645,7 +4682,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4661,14 +4698,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4734,7 +4771,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4750,14 +4787,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4813,7 +4850,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4829,14 +4866,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -7023,7 +7060,7 @@ <h3 id="openml.extensions.extension_interface.Extension.seed_model" class="doc d
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -7033,7 +7070,7 @@ <h3 id="openml.extensions.extension_interface.Extension.seed_model" class="doc d
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/extensions/functions/index.html b/reference/extensions/functions/index.html
index 12af1a8..9c452d6 100644
--- a/reference/extensions/functions/index.html
+++ b/reference/extensions/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4541,6 +4558,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4558,7 +4595,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4574,14 +4611,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4647,7 +4684,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4663,14 +4700,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4726,7 +4763,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4742,14 +4779,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5459,7 +5496,7 @@ <h2 id="openml.extensions.functions.register_extension" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5469,7 +5506,7 @@ <h2 id="openml.extensions.functions.register_extension" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/extensions/index.html b/reference/extensions/index.html
index 09dae7f..247bf28 100644
--- a/reference/extensions/index.html
+++ b/reference/extensions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4469,6 +4486,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4486,7 +4523,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4502,14 +4539,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4575,7 +4612,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4591,14 +4628,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4654,7 +4691,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4670,14 +4707,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -7329,7 +7366,7 @@ <h2 id="openml.extensions.register_extension" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -7339,7 +7376,7 @@ <h2 id="openml.extensions.register_extension" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/extensions/sklearn/extension/index.html b/reference/extensions/sklearn/extension/index.html
index 0c7ad00..d4e3a69 100644
--- a/reference/extensions/sklearn/extension/index.html
+++ b/reference/extensions/sklearn/extension/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4639,6 +4656,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4656,7 +4693,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4672,14 +4709,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4745,7 +4782,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4761,14 +4798,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4824,7 +4861,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4840,14 +4877,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -11812,7 +11849,7 @@ <h3 id="openml.extensions.sklearn.extension.SklearnExtension.trim_flow_name" cla
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -11822,7 +11859,7 @@ <h3 id="openml.extensions.sklearn.extension.SklearnExtension.trim_flow_name" cla
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/extensions/sklearn/index.html b/reference/extensions/sklearn/index.html
index 8aad299..b4bf530 100644
--- a/reference/extensions/sklearn/index.html
+++ b/reference/extensions/sklearn/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4471,6 +4488,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4488,7 +4525,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4504,14 +4541,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4577,7 +4614,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4593,14 +4630,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4656,7 +4693,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4672,14 +4709,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -11760,7 +11797,7 @@ <h2 id="openml.extensions.sklearn.cont" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -11770,7 +11807,7 @@ <h2 id="openml.extensions.sklearn.cont" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/flows/flow/index.html b/reference/flows/flow/index.html
index 3df58a8..a495c15 100644
--- a/reference/flows/flow/index.html
+++ b/reference/flows/flow/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4592,6 +4609,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4609,7 +4646,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4625,14 +4662,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4698,7 +4735,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4714,14 +4751,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4777,7 +4814,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4793,14 +4830,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -7144,7 +7181,7 @@ <h3 id="openml.flows.flow.OpenMLFlow.to_filesystem" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -7154,7 +7191,7 @@ <h3 id="openml.flows.flow.OpenMLFlow.to_filesystem" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/flows/functions/index.html b/reference/flows/functions/index.html
index 3ca4d42..5f05704 100644
--- a/reference/flows/functions/index.html
+++ b/reference/flows/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4577,6 +4594,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4594,7 +4631,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4610,14 +4647,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4683,7 +4720,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4699,14 +4736,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4762,7 +4799,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4778,14 +4815,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -6741,7 +6778,7 @@ <h2 id="openml.flows.functions.list_flows" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -6751,7 +6788,7 @@ <h2 id="openml.flows.functions.list_flows" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/flows/index.html b/reference/flows/index.html
index 4f75e0d..bc88bfe 100644
--- a/reference/flows/index.html
+++ b/reference/flows/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4469,6 +4486,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4486,7 +4523,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4502,14 +4539,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4575,7 +4612,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4591,14 +4628,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4654,7 +4691,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4670,14 +4707,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -8542,7 +8579,7 @@ <h2 id="openml.flows.list_flows" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -8552,7 +8589,7 @@ <h2 id="openml.flows.list_flows" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/index.html b/reference/index.html
index 0abdd7d..d36722a 100644
--- a/reference/index.html
+++ b/reference/index.html
@@ -367,6 +367,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4454,6 +4471,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4471,7 +4508,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4487,14 +4524,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4560,7 +4597,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4576,14 +4613,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -19462,7 +19499,7 @@ <h2 id="openml.populate_cache" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -19472,7 +19509,7 @@ <h2 id="openml.populate_cache" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/runs/functions/index.html b/reference/runs/functions/index.html
index f56d5b4..e2f522b 100644
--- a/reference/runs/functions/index.html
+++ b/reference/runs/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4622,6 +4639,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4639,7 +4676,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4655,14 +4692,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4728,7 +4765,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4744,14 +4781,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4807,7 +4844,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4823,14 +4860,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -8098,7 +8135,7 @@ <h2 id="openml.runs.functions.run_model_on_task" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -8108,7 +8145,7 @@ <h2 id="openml.runs.functions.run_model_on_task" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/runs/index.html b/reference/runs/index.html
index 3c307a0..3913259 100644
--- a/reference/runs/index.html
+++ b/reference/runs/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4469,6 +4486,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4486,7 +4523,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4502,14 +4539,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4575,7 +4612,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4591,14 +4628,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4654,7 +4691,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4670,14 +4707,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -12684,7 +12721,7 @@ <h2 id="openml.runs.run_model_on_task" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -12694,7 +12731,7 @@ <h2 id="openml.runs.run_model_on_task" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/runs/run/index.html b/reference/runs/run/index.html
index 0db0bfe..93428c0 100644
--- a/reference/runs/run/index.html
+++ b/reference/runs/run/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4574,6 +4591,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4591,7 +4628,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4607,14 +4644,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4680,7 +4717,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4696,14 +4733,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4759,7 +4796,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4775,14 +4812,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -7670,7 +7707,7 @@ <h3 id="openml.runs.run.OpenMLRun.to_filesystem" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -7680,7 +7717,7 @@ <h3 id="openml.runs.run.OpenMLRun.to_filesystem" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/runs/trace/index.html b/reference/runs/trace/index.html
index 7f05628..e708565 100644
--- a/reference/runs/trace/index.html
+++ b/reference/runs/trace/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4616,6 +4633,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4633,7 +4670,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4649,14 +4686,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4722,7 +4759,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4738,14 +4775,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4801,7 +4838,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4817,14 +4854,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -7623,7 +7660,7 @@ <h3 id="openml.runs.trace.OpenMLTraceIteration.get_parameters" class="doc doc-he
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -7633,7 +7670,7 @@ <h3 id="openml.runs.trace.OpenMLTraceIteration.get_parameters" class="doc doc-he
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/setups/functions/index.html b/reference/setups/functions/index.html
index 352cffe..b952ee3 100644
--- a/reference/setups/functions/index.html
+++ b/reference/setups/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4559,6 +4576,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4576,7 +4613,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4592,14 +4629,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4665,7 +4702,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4681,14 +4718,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4744,7 +4781,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4760,14 +4797,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5857,7 +5894,7 @@ <h2 id="openml.setups.functions.setup_exists" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5867,7 +5904,7 @@ <h2 id="openml.setups.functions.setup_exists" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/setups/index.html b/reference/setups/index.html
index 5ec4cdb..d610898 100644
--- a/reference/setups/index.html
+++ b/reference/setups/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4469,6 +4486,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4486,7 +4523,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4502,14 +4539,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4575,7 +4612,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4591,14 +4628,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4654,7 +4691,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4670,14 +4707,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -6228,7 +6265,7 @@ <h2 id="openml.setups.setup_exists" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -6238,7 +6275,7 @@ <h2 id="openml.setups.setup_exists" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/setups/setup/index.html b/reference/setups/setup/index.html
index 350bfce..d8ae228 100644
--- a/reference/setups/setup/index.html
+++ b/reference/setups/setup/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4532,6 +4549,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4549,7 +4586,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4565,14 +4602,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4638,7 +4675,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4654,14 +4691,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4717,7 +4754,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4733,14 +4770,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5563,7 +5600,7 @@ <h2 id="openml.setups.setup.OpenMLSetup" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5573,7 +5610,7 @@ <h2 id="openml.setups.setup.OpenMLSetup" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/study/functions/index.html b/reference/study/functions/index.html
index 1fbcfbd..c5eeef8 100644
--- a/reference/study/functions/index.html
+++ b/reference/study/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4649,6 +4666,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4666,7 +4703,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4682,14 +4719,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4755,7 +4792,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4771,14 +4808,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4834,7 +4871,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4850,14 +4887,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -7790,7 +7827,7 @@ <h2 id="openml.study.functions.update_suite_status" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -7800,7 +7837,7 @@ <h2 id="openml.study.functions.update_suite_status" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/study/index.html b/reference/study/index.html
index 2d114a4..a556ad0 100644
--- a/reference/study/index.html
+++ b/reference/study/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4469,6 +4486,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4486,7 +4523,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4502,14 +4539,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4575,7 +4612,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4591,14 +4628,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4654,7 +4691,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4670,14 +4707,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -8240,7 +8277,7 @@ <h2 id="openml.study.update_suite_status" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -8250,7 +8287,7 @@ <h2 id="openml.study.update_suite_status" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/study/study/index.html b/reference/study/study/index.html
index 962ddb5..69332bc 100644
--- a/reference/study/study/index.html
+++ b/reference/study/study/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4574,6 +4591,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4591,7 +4628,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4607,14 +4644,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4680,7 +4717,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4696,14 +4733,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4759,7 +4796,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4775,14 +4812,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -6642,7 +6679,7 @@ <h2 id="openml.study.study.OpenMLStudy" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -6652,7 +6689,7 @@ <h2 id="openml.study.study.OpenMLStudy" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/tasks/functions/index.html b/reference/tasks/functions/index.html
index 314bea7..cc4afd3 100644
--- a/reference/tasks/functions/index.html
+++ b/reference/tasks/functions/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4568,6 +4585,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4585,7 +4622,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4601,14 +4638,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4674,7 +4711,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4690,14 +4727,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4753,7 +4790,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4769,14 +4806,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -6632,7 +6669,7 @@ <h2 id="openml.tasks.functions.list_tasks" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -6642,7 +6679,7 @@ <h2 id="openml.tasks.functions.list_tasks" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/tasks/index.html b/reference/tasks/index.html
index 1fe174f..0391fb2 100644
--- a/reference/tasks/index.html
+++ b/reference/tasks/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4469,6 +4486,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4486,7 +4523,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4502,14 +4539,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4575,7 +4612,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4591,14 +4628,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4654,7 +4691,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4670,14 +4707,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -10095,7 +10132,7 @@ <h2 id="openml.tasks.list_tasks" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -10105,7 +10142,7 @@ <h2 id="openml.tasks.list_tasks" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/tasks/split/index.html b/reference/tasks/split/index.html
index 083e958..77ffa91 100644
--- a/reference/tasks/split/index.html
+++ b/reference/tasks/split/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4547,6 +4564,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4564,7 +4601,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4580,14 +4617,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4653,7 +4690,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4669,14 +4706,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4732,7 +4769,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4748,14 +4785,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5715,7 +5752,7 @@ <h2 id="openml.tasks.split.Split" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5725,7 +5762,7 @@ <h2 id="openml.tasks.split.Split" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/tasks/task/index.html b/reference/tasks/task/index.html
index f445e23..960b973 100644
--- a/reference/tasks/task/index.html
+++ b/reference/tasks/task/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4667,6 +4684,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4684,7 +4721,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4700,14 +4737,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4773,7 +4810,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4789,14 +4826,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4852,7 +4889,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4868,14 +4905,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -8380,7 +8417,7 @@ <h2 id="openml.tasks.task.TaskType" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -8390,7 +8427,7 @@ <h2 id="openml.tasks.task.TaskType" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/testing/index.html b/reference/testing/index.html
index 1624951..4efc831 100644
--- a/reference/testing/index.html
+++ b/reference/testing/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4563,6 +4580,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4580,7 +4617,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4596,14 +4633,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4669,7 +4706,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4685,14 +4722,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4748,7 +4785,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4764,14 +4801,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5982,7 +6019,7 @@ <h2 id="openml.testing.check_task_existence" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5992,7 +6029,7 @@ <h2 id="openml.testing.check_task_existence" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/reference/utils/index.html b/reference/utils/index.html
index b1b7bec..16eb28f 100644
--- a/reference/utils/index.html
+++ b/reference/utils/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4521,6 +4538,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4538,7 +4575,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4554,14 +4591,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4627,7 +4664,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4643,14 +4680,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4706,7 +4743,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4722,14 +4759,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5169,7 +5206,7 @@ <h2 id="openml.utils.extract_xml_tags" class="doc doc-heading">
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5179,7 +5216,7 @@ <h2 id="openml.utils.extract_xml_tags" class="doc doc-heading">
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/search/search_index.json b/search/search_index.json
index 718ff8e..cf625d2 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-\\.]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Introduction","text":"An open, automated, and frictionless machine learning environment. <p> \u00a0 1000s of data sets, uniformly formatted, easy to load, organized online</p> <p> \u00a0Models and pipelines automatically uploaded from machine learning libraries</p> <p> Extensive APIs to integrate OpenML into your tools and scripts</p> <p>  Easily reproducible results (e.g. models, evaluations) for comparison and reuse</p> <p>\u00a0 Stand on the shoulders of giants, and collaborate in real time</p> <p>\u00a0 Make your work more visible and reusable</p> <p>\u00a0 Built for automation: streamline your experiments and model building</p>"},{"location":"#installation","title":"Installation","text":"<p>The OpenML package is available in many languages and across libraries. For more information about them, see the Integrations page.</p> Python/sklearnPytorchKerasTensorFlowRJuliaRUST.Net <ul> <li>Python/sklearn repository</li> <li><code>pip install openml</code></li> </ul> <ul> <li>Pytorch repository</li> <li><code>pip install openml-pytorch</code></li> </ul> <ul> <li>Keras repository</li> <li><code>pip install openml-keras</code></li> </ul> <ul> <li>TensorFlow repository</li> <li><code>pip install openml-tensorflow</code></li> </ul> <ul> <li>R repository</li> <li><code>install.packages(\"mlr3oml\")</code></li> </ul> <ul> <li>Julia repository</li> <li><code>using Pkg;Pkg.add(\"OpenML\")</code></li> </ul> <ul> <li>RUST repository</li> <li>Install from source</li> </ul> <ul> <li>.Net repository</li> <li><code>Install-Package openMl</code></li> </ul> <p>You might also need to set up the API key. For more information, see Authentication.</p>"},{"location":"#learning-openml","title":"Learning OpenML","text":"<p>Aside from the individual package documentations, you can learn more about OpenML through the following resources: The core concepts of OpenML are explained in the Concepts page. These concepts include the principle behind using Datasets, Runs, Tasks, Flows, Benchmarking and much more. Going through them will help you leverage OpenML even better in your work.</p>"},{"location":"#contributing-to-openml","title":"Contributing to OpenML","text":"<p>OpenML is an open source project, hosted on GitHub. We welcome everybody to help improve OpenML, and make it more useful for everyone. For more information on how to contribute, see the Contributing page.</p> <p>We want to make machine learning and data analysis simple, accessible, collaborative and open with an optimal division of labour between computers and humans.</p>"},{"location":"#want-to-get-involved","title":"Want to get involved?","text":"<p>Awesome, we're happy to have you! </p> <p>OpenML is dependent on the community. If you want to help, please email us (openmlHQ@googlegroups.com). If you feel already comfortable you can help by opening issues or make a pull request on GitHub. We also have regular workshops you can join (they are announced on openml.org).</p>"},{"location":"apiv2/","title":"OpenML Server API Software Documentation","text":"<p>This is the Python-based OpenML REST API server. It's a rewrite of our old backend built with a modern Python-based stack.</p> <p>Looking to access data on OpenML?</p> <p>If you simply want to access data stored on OpenML in a programmatic way, please have a look at connector packages in Python, Java, or R.</p> <p>If you are looking to interface directly with the REST API, and are looking for documentation on the REST API endpoints, visit the APIs page.</p> <p>This documentation is for developing and hosting your own OpenML REST API.</p>"},{"location":"apiv2/#development-roadmap","title":"Development Roadmap","text":"<p>First we will mimic current server functionality, relying on many implementation details present in the current production server. We will implement all endpoints using the SQL text queries based on PHP implementation, which should give near-identical responses to the current JSON endpoints. Minor exceptions are permitted but will be documented.</p> <p>At the same time we may also provide a work-in-progress \"new\" endpoint, but there won't be official support for it at this stage. After we verify the output of the endpoints are identical (minus any intentional documented differences), we will officially release the new API. The old API will remain available. After that, we can start working on a new version of the JSON API which is more standardized, leverages typing, and so on:</p> <ul> <li>Clean up the database: standardize value formats where possible (e.g., (un)quoting    contributor names in the dataset's contributor field), and add database level    constraints on new values.</li> <li>Redesign what the new API responses should look like and implement them,    API will be available to the public as it is developed.</li> <li>Refactor code-base to use ORM (using <code>SQLAlchemy</code>, <code>SQLModel</code>, or similar).</li> <li>Officially release the modernized API.</li> </ul> <p>There is no planned sunset date for the old API. This will depend on the progress with the new API as well as the usage numbers of the old API.</p>"},{"location":"apiv2/installation/","title":"Installation","text":"<p>Current instructions tested on Mac, but likely work on most Unix systems.</p> <p>The OpenML server will be developed and maintained for the latest minor release of Python (Python 3.12 as of writing). You can install the dependencies locally or work with docker containers.</p> Use <code>pyenv</code> to manage Python installations <p>We recommend using <code>pyenv</code> if you are working with multiple local Python versions. After following the installation instructions for <code>pyenv</code> check that you can execute it:</p> <pre><code>&gt; pyenv local\n3.12\n</code></pre> <p>If <code>pyenv</code> can't be found, please make sure to update the terminal environment (either by <code>reset</code>ing it, or by closing and opening the terminal). If you get the message <code>pyenv: no local version configured for this directory</code> first clone the repository as described below and try again from the root of the cloned repository.</p> <p>You can then install the Python version this project uses with: <code>cat .python-version | pyenv install</code></p>"},{"location":"apiv2/installation/#local-installation","title":"Local Installation","text":"<p>These instructions assume Python 3.12 and git are already installed.</p> <p>You may need to install Python3 and MySQL development headers.</p> <p>It may be necessary to first install additional headers before proceeding with a local installation of the <code>mysqlclient</code> dependency. They are documented under \"Installation\" of the <code>mysqlclient</code> documentation.</p> For UsersFor Contributors <p>If you don't plan to make code changes, you can install directly from Github. We recommend to install the OpenML server and its dependencies into a new virtual environment. Installing the project into a new virtual environment<pre><code>python -m venv venv\nsource venv/bin/activate\n\npython -m pip install git+https://github.com/openml/server-api.git\n</code></pre> If you do plan to make code changes, we recommend you follow the instructions under the \"For Contributors\" tab, even if you do not plan to contribute your changes back into the project.</p> <p>If you plan to make changes to this project, it will be useful to install the project from a cloned fork. To fork the project, go to our project page and click \"fork\". This makes a copy of the repository under your own Github account. You can then clone your own fork (replace <code>USER_NAME</code> with your Github username):</p> Cloning your fork<pre><code>git clone https://github.com/USER_NAME/server-api.git\ncd server-api\n</code></pre> <p>Then we can install the project into a new virtual environment in edit mode:</p> <p>Installing the project into a new virtual environment<pre><code>python -m venv venv\nsource venv/bin/activate\n\npython -m pip install -e \".[dev,docs]\"\n</code></pre> Note that this also installs optional dependencies for development and documentation tools. We require this for contributors, but we also highly recommend it anyone that plans to make code changes.</p>"},{"location":"apiv2/installation/#setting-up-a-database-server","title":"Setting up a Database Server","text":"<p>Depending on your use of the server, there are multiple ways to set up your own OpenML database. To simply connect to an existing database, see configuring the REST API Server below.</p>"},{"location":"apiv2/installation/#setting-up-a-new-database","title":"Setting up a new database","text":"<p>This sets up an entirely empty database with the expected OpenML tables in place. This is intended for new deployments of OpenML, for example to host a private OpenML server.</p> <p>Instructions are incomplete. See issue#78.</p>"},{"location":"apiv2/installation/#setting-up-a-test-database","title":"Setting up a test database","text":"<p>We provide a prebuilt docker image that already contains test data.</p> Docker ComposeDocker Run <p>To start the database through <code>docker compose</code>, run:</p> <pre><code>docker compose up database\n</code></pre> <p>which starts a database.</p> <p>To start a test database as stand-alone container, run:</p> <pre><code>docker run  --rm -e MYSQL_ROOT_PASSWORD=ok -p 3306:3306 -d --name openml-test-database openml/test-database:latest\n</code></pre> <p>You may opt to add the container to a network instead, to make it reachable from other docker containers:</p> <pre><code>docker network create openml\ndocker run  --rm -e MYSQL_ROOT_PASSWORD=ok -p 3306:3306 -d --name openml-test-database --network openml openml/test-database:latest\n</code></pre> <p>The container may take a minute to initialise, but afterwards you can connect to it. Either from a local <code>mysql</code> client at <code>127.0.0.1:3306</code> or from a docker container on the same network. For example:</p> <p><pre><code>docker run --network NETWORK --rm -it mysql mysql -hopenml-test-database -uroot -pok\n</code></pre> where <code>NETWORK</code> is <code>openml</code> when using <code>docker run</code> when following the example, and <code>NETWORK</code> is <code>server-api_default</code> if you used <code>docker compose</code> (specifically, it is <code>DIRECTORY_NAME</code> + <code>_default</code>, so if you renamed the <code>server-api</code> directory to something else, the network name reflects that).</p>"},{"location":"apiv2/installation/#configuring-the-rest-api-server","title":"Configuring the REST API Server","text":"<p>The REST API is configured through a TOML file.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/migration/","title":"Migration","text":"<p>The Python reimplementation provides the same endpoints as the old API, which are largely functioning the same way. However, there are a few major deviations:</p> <ul> <li>Use of typed JSON: e.g., when a value represents an integer, it is returned as integer.</li> <li>Lists when multiple values are possible: if a field can have none, one, or multiple entries (e.g., authors), we always return a list.</li> <li>Restriction or expansion of input types as appropriate.</li> <li>Standardizing authentication and access messages, and consistently execute those checks   before fetching data or providing error messages about the data.</li> </ul> <p>The list above is not exhaustive. Minor changes include, for example, bug fixes and the removal of unnecessary nesting. There may be undocumented changes, especially in edge cases which may not have occurred in the test environment. As the PHP API was underspecified, the re-implementation is based on a mix of reading old code and probing the API. If there is a behavioral change which was not documented but affects you, please open a bug report.</p>"},{"location":"apiv2/migration/#all-endpoints","title":"All Endpoints","text":"<p>The following changes affect all endpoints.</p>"},{"location":"apiv2/migration/#error-on-invalid-input","title":"Error on Invalid Input","text":"<p>When providing input of invalid types (e.g., a non-integer dataset id) the HTTP header and JSON content will be different.</p> HTTP Header<pre><code>- 412 Precondition Failed\n+ 422 Unprocessable Entity\n</code></pre> JSON Content<pre><code>- {\"error\":{\"code\":\"100\",\"message\":\"Function not valid\"}}\n+ {\"detail\":[{\"loc\":[\"query\",\"_dataset_id\"],\"msg\":\"value is not a valid integer\",\"type\":\"type_error.integer\"}]}\n</code></pre> <p>Input validation has been added to many end points</p> <p>There are endpoints which previously did not do any input validation.    These endpoints now do enforce stricter input constraints.    Constraints for each endpoint parameter are documented in the API docs.</p>"},{"location":"apiv2/migration/#other-errors","title":"Other Errors","text":"<p>For any other error messages, the response is identical except that outer field will be <code>\"detail\"</code> instead of <code>\"error\"</code>:</p> JSON Content<pre><code>- {\"error\":{\"code\":\"112\",\"message\":\"No access granted\"}}\n+ {\"detail\":{\"code\":\"112\",\"message\":\"No access granted\"}}\n</code></pre> <p>In some cases the JSON endpoints previously returned XML (example), the new API always returns JSON.</p> XML replaced by JSON<pre><code>- &lt;oml:error xmlns:oml=\"http://openml.org/openml\"&gt;\n-   &lt;oml:code&gt;103&lt;/oml:code&gt;\n-   &lt;oml:message&gt;Authentication failed&lt;/oml:message&gt;\n- &lt;/oml:error&gt;\n+ {\"detail\": {\"code\":\"103\", \"message\": \"Authentication failed\" } }\n</code></pre>"},{"location":"apiv2/migration/#datasets","title":"Datasets","text":""},{"location":"apiv2/migration/#get-dataset_id","title":"<code>GET /{dataset_id}</code>","text":"<ul> <li>Dataset format names are normalized to be all lower-case    (<code>\"Sparse_ARFF\"</code> -&gt;  <code>\"sparse_arff\"</code>).</li> <li>Non-<code>arff</code> datasets will not incorrectly have a <code>\"parquet_url\"</code> (openml#1189).</li> <li>If <code>\"creator\"</code> contains multiple comma-separated creators it is always returned    as a list, instead of it depending on the quotation used by the original uploader.</li> <li>For (some?) datasets that have multiple values in <code>\"ignore_attribute\"</code>, this field    is correctly populated instead of omitted.</li> <li>Processing date is formatted with a <code>T</code> in the middle:   processing_date<pre><code>- \"2019-07-09 15:22:03\"\n+ \"2019-07-09T15:22:03\"\n</code></pre></li> <li>Fields which may contain lists of values (e.g., <code>creator</code>, <code>contributor</code>) now always   returns a list (which may also be empty or contain a single element).</li> <li>Fields without a set value are no longer automatically removed from the response.</li> </ul>"},{"location":"apiv2/migration/#get-datalistfilters","title":"<code>GET /data/list/{filters}</code>","text":"<p>The endpoint now accepts the filters in the body of the request, instead of as query parameters. <pre><code>-  curl -d '' 127.0.0.1:8002/api/v1/json/data/list/status/active\n+ curl -X 'POST' 'http://localhost:8001/v1/datasets/list' \\\n+  -H 'Content-Type: application/json' \\\n+  -d '{}'\n</code></pre> This endpoint is now also available via a <code>POST</code> request, and will exhibit the same behavior regardless of how it is accessed.</p> <p>When accessing this endpoint when authenticated as administrator, it now correctly includes datasets which are private.</p> <p>The <code>limit</code> and <code>offset</code> parameters can now be used independently, you no longer need to provide both if you wish to set only one.</p>"},{"location":"apiv2/migration/#post-datasetstag","title":"<code>POST /datasets/tag</code>","text":"<p>When successful, the \"tag\" property in the returned response is now always a list, even if only one tag exists for the entity. For example, after tagging dataset 21 with the tag <code>\"foo\"</code>: <pre><code>{\n   data_tag\": {\n      \"id\": \"21\",\n-      \"tag\": \"foo\"\n+      \"tag\": [\"foo\"]\n   }\n}\n</code></pre></p>"},{"location":"apiv2/migration/#studies","title":"Studies","text":""},{"location":"apiv2/migration/#get-id_or_alias","title":"<code>GET /{id_or_alias}</code>","text":"<p>Old-style \"legacy\" studies which are solely based on tags are no longer supported.</p> Affected Legacy Studies <p>Only 24 old studies were affected by this change, listed below. There is currently not yet a migration plan for these studies.</p> id name 1 A large-scale comparison of classification algorit... 2 Fast Algorithm Selection using Learning Curves 3 Multi-Task Learning with a Natural Metric for Quan... 5 Local and Global Feature Selection on Multilabel T... 7 Massive machine learning experiments using mlr and... 8 Decision tree comparaison 10 Collaborative primer 11 Having a Blast: Meta-Learning and Heterogeneous En... 12 Subspace Clustering via Seeking Neighbors with Min... 13 Meta-QSAR: learning how to learn QSARs 17 Subgroup Discovery 20 Mythbusting data mining urban legends through larg... 22 Identifying critical paths in undergraduate progra... 24 OpenML R paper 25 Bernd Demo Study for Multiclass SVMs OML WS 2016 27 Compare three different SVM versions of R package ... 30 OpenML Paper Study 31 Iris Data set Study 32 Data Streams and more 34 Massively Collaborative Machine Learning 37 Speeding up Algorithm Selection via Meta-learning ... 38 Performance of new ctree implementations on classi... 41 ASLib OpenML Scenario 50 Hyper-parameter tuning of Decision Trees 51 ensemble on diabetes"},{"location":"apiv2/migration/#flows","title":"Flows","text":""},{"location":"apiv2/migration/#get-flowexistsnameexternal_version","title":"<code>GET /flow/exists/{name}/{external_version}/</code>","text":"<p>Behavior has changed slightly. When a flow is found:</p> <pre><code>- { \"flow_exists\": { \"exists\": \"true\", \"flow_id\": \"123\" } }\n+ { \"flow_id\": 123 }\n</code></pre> <p>and when a flow is not found: <pre><code>- { \"flow_exists\": { \"exists\": \"false\", \"flow_id\": \"-1\" } }\n+ { \"detail\": \"Flow not found.\" }\n</code></pre> and the HTTP header status code is <code>404</code> (NOT FOUND) instead of <code>200</code> (OK).</p> <p>In the future the successful case will more likely just return the flow immediately instead (see #170).</p>"},{"location":"apiv2/migration/#others","title":"Others","text":""},{"location":"apiv2/migration/#get-estimationprocedurelist","title":"<code>GET /estimationprocedure/list</code>","text":"<p>The <code>ttid</code> field has been renamed to <code>task_type_id</code>. All values are now typed. Outer levels of nesting have been removed.</p>"},{"location":"apiv2/migration/#get-evaluationmeasureslist","title":"<code>GET /evaluationmeasures/list</code>","text":"<p>Outer levels of nesting have been removed.</p>"},{"location":"apiv2/contributing/","title":"Contributing","text":"<p>Like many other open source projects, contributions from the community are an essential piece in making OpenML successful. We very much appreciate that you want to make a contribution. To try to make this as smooth as possible for everyone involved, we wrote this contribution guide. There are many kinds of contributions, such as bug reports, participating in discussions, making code contributions, or updating documentation.</p>"},{"location":"apiv2/contributing/#bug-reports","title":"Bug reports","text":"<p>When you encounter an issue using or deploying the Python-based REST API, the first step is to search our issue tracker to see if others have already reported the issue. If this is the case, please indicate that you encountered the issue by leaving a  on the issue. Feel free to leave a comment if you have additional information that may be useful to track down or fix the bug.</p> <p>If the bug has not been reported before, please open a new issue to report it. When you open a bug report, please include:</p> <ul> <li>a minimal reproducible example,</li> <li>a description of the expected behavior,</li> <li>a description of the encountered behavior,</li> <li>and any additional information that you think may be relevant, such as which environment you encountered it in, or when you first encountered the bug.</li> </ul>"},{"location":"apiv2/contributing/#code","title":"Code","text":"<p>If you want to make code contributions, please first make sure there is an open issue that describes what the contribution should look like. This may be a proposal that you write yourself, or an existing open issue that you want to help us with. Please indicate on that issue that you would like to contribute to it. This way, we can ensure that everything is clear, it's not out of date, and \"officially\" assign you to the issue so that others know its being worked on. Making sure there is a clear description and clear assignment helps prevent a situation where someone makes a large contribution that is unwanted, or is simultaneously developed by someone else. With an issue assigned, please head over to the \"Development\" section.</p>"},{"location":"apiv2/contributing/#documentation","title":"Documentation","text":"<p>For minor fixes, it's fine to make the changes and submit them through a pull request. For larger changes, please make sure you are assigned to an issue first as is described in the \"Code\" section of this page. Then, visit the \"Documentation\" page to learn how to contribute documentation changes.</p>"},{"location":"apiv2/contributing/#other","title":"Other","text":"<p>If you are looking to contribute to OpenML in general, visit \"Contributing\" on the OpenML website.</p>"},{"location":"apiv2/contributing/contributing/","title":"Setting up the development environment","text":"<p>First, follow the installation instructions for contributors to install a local fork with optional development dependencies. Stop when you reach the section \"Setting up a Database Server\".</p>"},{"location":"apiv2/contributing/contributing/#pre-commit","title":"Pre-commit","text":"<p>We use <code>pre-commit</code> to ensure certain tools and checks are ran before each commit. These tools perform code-formatting, linting, and more. This makes the code more consistent across the board, which makes it easier to work with each other's code and also can catch common errors. After installing it, it will automatically run when you try to make a commit. Install it now and verify that all checks pass out of the box:</p> <p>Install pre-commit and verify it works<pre><code>pre-commit install\npre-commit run --all-files\n</code></pre> Running the tool the first time may be slow as tools need to be installed, and many tools will build a cache so subsequent runs are faster. Under normal circumstances, running the pre-commit chain shouldn't take more than a few seconds.</p>"},{"location":"apiv2/contributing/contributing/#docker","title":"Docker","text":"<p>With the projected forked, cloned, and installed, the easiest way to set up all required services for development is through <code>docker compose</code>.</p>"},{"location":"apiv2/contributing/contributing/#starting-containers","title":"Starting containers","text":"<pre><code>docker compose up\n</code></pre> <p>This will spin up 4 containers, as defined in the <code>docker-compose.yaml</code> file:</p> <ul> <li><code>openml-test-database</code>: this is a mysql database prepopulated with test data.     It is reachable from the host machine with port <code>3306</code>, by default it is configured     to have a root user with password <code>\"ok\"</code>.</li> <li><code>server-api-docs-1</code>: this container serves project documentation at <code>localhost:8000</code>.     These pages are built from the documents in the <code>docs/</code> directory of this repository,     whenever you edit and save a file there, the page will immediately be updated.</li> <li><code>server-api-php-api-1</code>: this container serves the old PHP REST API at <code>localhost:8002</code>.     For example, visit http://localhost:8002/api/v1/json/data/1     to fetch a JSON description of dataset 1.</li> <li><code>python-api</code>: this container serves the new Python-based REST API at <code>localhost:8001</code>.     For example, visit http://localhost:8001/docs to see     the REST API documentation. Changes to the code in <code>src/</code> will be reflected in this     container.</li> </ul> <p>You don't always need every container, often just having a database and the Python-based REST API may be enough. In that case, only specify those services:</p> <pre><code>docker compose up database python-api\n</code></pre> <p>Refer to the <code>docker compose</code> documentation for more uses.</p>"},{"location":"apiv2/contributing/contributing/#connecting-to-containers","title":"Connecting to containers","text":"<p>To connect to a container, run:</p> <pre><code>docker exec -it CONTAINER_NAME /bin/bash\n</code></pre> <p>where <code>CONTAINER_NAME</code> is the name of the container. If you are unsure of your container name, then <code>docker container ls</code> may help you find it. Assuming the default container names are used, you may connect to the Python-based REST API container using:</p> <pre><code>docker exec -it python-api /bin/bash\n</code></pre> <p>This is useful, for example, to run unit tests in the container:</p> <pre><code>python -m pytest -x -v -m \"not web\"\n</code></pre>"},{"location":"apiv2/contributing/contributing/#unit-tests","title":"Unit tests","text":"<p>Our unit tests are written with the <code>pytest</code> framework. An invocation could look like this:</p> <pre><code>python -m pytest -v -x --lf -m \"not web\"\n</code></pre> <p>Where <code>-v</code> show the name of each test ran, <code>-x</code> ensures testing stops on first failure, <code>--lf</code> will first run the test(s) which failed last, and <code>-m \"not web\"</code> specifies which tests (not) to run.</p> <p>The directory structure of our tests follows the structure of the <code>src/</code> directory. For files, we follow the convention of appending <code>_test</code>. Try to keep tests as small as possible, and only rely on database and/or web connections when absolutely necessary.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/contributing/contributing/#yaml-validation","title":"YAML validation","text":"<p>The project contains various <code>yaml</code> files, for example to configure <code>mkdocs</code> or to describe Github workflows. For these <code>yaml</code> files we can configure automatic schema validation, to ensure that the files are valid without having to run the server. This also helps with other IDE features like autocomplete. Setting this up is not required, but incredibly useful if you do need to edit these files. The following <code>yaml</code> files have schemas:</p> File(s) Schema URL mkdocs.yml https://squidfunk.github.io/mkdocs-material/schema.json .pre-commit-config.yaml https://json.schemastore.org/pre-commit-config.json .github/workflows/*.yaml https://json.schemastore.org/github-workflow PyCharmVSCode <p>In PyCharm, these can be configured from <code>settings</code> &gt; <code>languages &amp; frameworks</code> &gt; <code>Schemas and DTDs</code> &gt; <code>JSON Schema Mappings</code>. There, add mappings per file or file pattern.</p> <p>In VSCode, these can be configured from <code>settings</code> &gt; <code>Extetions</code> &gt; <code>JSON</code> &gt; <code>Edit in settings.json</code>. There, add mappings per file or file pattern. For example:</p> <pre><code>\"json.schemas\": [\n   {\n         \"fileMatch\": [\n            \"/myfile\"\n         ],\n         \"url\": \"schemaURL\"\n   }\n\n]\n</code></pre>"},{"location":"apiv2/contributing/contributing/#connecting-to-another-database","title":"Connecting to another database","text":"<p>In addition to the database setup described in the installation guide, we also host a database on our server which may be connected to that is available to OpenML core contributors. If you are a core contributor and need access, please reach out to one of the engineers in Eindhoven.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/contributing/documentation/","title":"Documentation with mkdocs-material","text":"<p>Our documentation is built using mkdocs and the mkdocs-material theme. We also use some plugins. Please refer to the \"Getting Started\" pages of <code>mkdocs-material</code> for a general overview on how to work with <code>mkdocs-material</code>. All documentation files are in the <code>docs/</code> folder, except for the configuration file which is <code>mkdocs.yml</code> at the root of the repository.</p> <p>For minor changes, it should be fine to edit the page directly on Github. That should commit to a separate branch (or fork), and you can set up a pull request. For larger changes, clone a fork of the repository as described in the \"Local Installation\" section.</p> DockerLocal installation <p>After cloning the repository, you may also build and serve the documentation through Docker:</p> <pre><code>docker compose up docs\n</code></pre> <p>Instead of installing all dependencies (with <code>python -m pip install -e \".[docs]\"</code>), you may also install just the documentation dependencies:</p> <pre><code>python -m pip install mkdocs-material mkdocs-section-index\n</code></pre> <p>You can then build and serve the documentation with</p> <pre><code>python -m mkdocs serve\n</code></pre> <p>This will serve the documentation from the <code>docs/</code> directory to http://localhost:8000/. Any updates you make to files in that directory will be reflected on the website. When you are happy with your changes, just commit and set up a pull request!</p>"},{"location":"apiv2/contributing/project_overview/","title":"Project overview","text":"<p>The Python-based REST API serves several groups of endpoints:</p> <ul> <li><code>/old/</code>: serves the old-style JSON format, this should mimic the PHP responses exactly with the only deviations recorded in the migration guide.</li> <li><code>/mldcat_ap/</code>: serves datasets in MLDCAT_AP format.</li> <li><code>/*</code>: serves new-style JSON format. At this point it is intentionally similar to the old-style format.</li> </ul> <p>The endpoints are specified in subdirectories of <code>src/routers</code>. They pull data from the database through the <code>src/database</code> module. The schemas for each entity, and possible conversions between them, are defined in the <code>src/schemas</code> directory.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/contributing/tests/","title":"Writing Tests","text":"<p>tl;dr:  - Setting up the <code>py_api</code> fixture to test directly against a REST API endpoint is really slow, only use it for migration/integration tests.  - Getting a database fixture and doing a database call is slow, consider mocking if appropriate.</p>"},{"location":"apiv2/contributing/tests/#overhead-from-fixtures","title":"Overhead from Fixtures","text":"<p>Sometimes, you want to interact with the REST API through the <code>py_api</code> fixture, or want access to a database with <code>user_test</code> or <code>expdb_test</code> fixtures. Be warned that these come with considerable relative overhead, which adds up when running thousands of tests.</p> <pre><code>@pytest.mark.parametrize('execution_number', range(5000))\ndef test_private_dataset_owner_access(\n        execution_number,\n        expdb_test: Connection,\n        user_test: Connection,\n        py_api: TestClient,\n) -&gt; None:\n    fetch_user(ApiKey.REGULAR_USER, user_test)  # accesses only the user db\n    get_estimation_procedures(expdb_test)  # accesses only the experiment db\n    py_api.get(\"/does/not/exist\")  # only queries the api\n    pass\n</code></pre> <p>When individually adding/removing components, we measure (for 5000 repeats, n=1):</p> expdb user api exp call user call api get time (s) \u274c \u274c \u274c \u274c \u274c \u274c 1.78 \u2705 \u274c \u274c \u274c \u274c \u274c 3.45 \u274c \u2705 \u274c \u274c \u274c \u274c 3.22 \u274c \u274c \u2705 \u274c \u274c \u274c 298.48 \u2705 \u2705 \u274c \u274c \u274c \u274c 4.44 \u2705 \u2705 \u2705 \u274c \u274c \u274c 285.69 \u2705 \u274c \u274c \u2705 \u274c \u274c 4.91 \u274c \u2705 \u274c \u274c \u2705 \u274c 5.81 \u2705 \u2705 \u2705 \u2705 \u2705 \u2705 307.91 <p>Adding a fixture that just returns some value adds only minimal overhead (1.91s), so the burden comes from establishing the database connection itself.</p> <p>We make the following observations:</p> <ul> <li>Adding a database fixture adds the same overhead as instantiating an entirely new test.</li> <li>Overhead of adding multiple database fixtures is not free</li> <li>The <code>py_api</code> fixture adds two orders of magnitude more overhead</li> </ul> <p>We want our tests to be fast, so we want to avoid using these fixtures when we reasonably can. We restrict usage of <code>py_api</code> fixtures to integration/migration tests, since it is very slow. These only run on CI before merges. For database fixtures</p> <p>We will write some fixtures that can be used to e.g., get a <code>User</code> without accessing the database. The validity of these users will be tested against the database in only a single test.</p>"},{"location":"apiv2/contributing/tests/#mocking","title":"Mocking","text":"<p>Mocking can help us reduce the reliance on database connections in tests. A mocked function can prevent accessing the database, and instead return a predefined value instead.</p> <p>It has a few upsides:  - It's faster than using a database fixture (see below).  - The test is not dependent on the database: you can run the test without a database.</p> <p>But it also has downsides:  - Behavior changes in the database, such as schema changes, are not automatically reflected in the tests.  - The database layer (e.g., queries) are not actually tested.</p> <p>Basically, the mocked behavior may not match real behavior when executed on a database. For this reason, for each mocked entity, we should add a test that verifies that if the database layer is invoked with the database, it returns the expected output that matches the mock. This is additional overhead in development, but hopefully it pays back in more granular test feedback and faster tests.</p> <p>On the speed of mocks, consider these two tests:</p> <p><pre><code>@pytest.mark.parametrize('execution_number', range(5000))\ndef test_private_dataset_owner_access(\n        execution_number,\n        admin,\n+        mocker,\n-        expdb_test: Connection,\n) -&gt; None:\n+    mock = mocker.patch('database.datasets.get')\n+    class Dataset(NamedTuple):\n+        uploader: int\n+        visibility: Visibility\n+    mock.return_value = Dataset(uploader=1, visibility=Visibility.PRIVATE)\n\n    _get_dataset_raise_otherwise(\n        dataset_id=1,\n        user=admin,\n-        expdb=expdb_test,\n+        expdb=None,\n    )\n</code></pre> There is only a single database call in the test. It fetches a record on an indexed field and does not require any joins. Despite the database call being very light, the database-included test is ~50% slower than the mocked version (3.50s vs 5.04s).</p>"},{"location":"benchmark/","title":"Benchmarking suites","text":"<p>Machine learning research depends on objectively interpretable, comparable, and reproducible algorithm benchmarks. OpenML aims to facilitate the creation of curated, comprehensive suites of machine learning tasks, covering precise sets of conditions.</p> <p>Seamlessly integrated into the OpenML platform, benchmark suites standardize the setup, execution, analysis, and reporting of benchmarks. Moreover, they make benchmarking a whole lot easier:  </p> <ul> <li> <p>all datasets are uniformly formatted in standardized data formats </p> </li> <li> <p>they can be easily downloaded programmatically through APIs and client libraries</p> </li> <li> <p>they come with machine-readable meta-information, such as the occurrence of missing values, to train algorithms correctly</p> </li> <li> <p>standardized train-test splits are provided to ensure that results can be objectively compared</p> </li> <li> <p>results can be shared in a reproducible way through the APIs</p> </li> <li> <p>results from other users can be easily downloaded and reused</p> </li> </ul>"},{"location":"benchmark/#software-interfaces","title":"Software interfaces","text":"<p>To use OpenML Benchmark suites, you can use bindings in several programming languages. These all interface with the OpenML REST API. The default endpoint for this is <code>https://www.openml.org/api/v1/</code>, but this can change when later versions of the API are released. To use the code examples below, you only need a recent version of one of the following libraries:</p> <ul> <li>OpenML Java ApiConnector (version <code>1.0.22</code> and up).</li> <li>OpenML Weka (version <code>0.9.6</code> and up). This package adds a Weka Integration.</li> <li>OpenML Python (version <code>0.9.0</code> and up)</li> <li>OpenML R (version <code>1.8</code> and up)</li> </ul>"},{"location":"benchmark/#using-openml-benchmark-suites","title":"Using OpenML Benchmark Suites","text":"<p>Below are walk-through instructions for common use cases, as well as code examples. These illustrations use the reference OpenML-CC18 benchmark suite, but you can replace it with any other benchmark suite. Note that a benchmark suite is a set of OpenML <code>tasks</code>, which envelop not only a specific dataset, but also the train-test splits and (for predictive tasks) the target feature.</p> Terminology and current status <p>Benchmark suites are sets of OpenML tasks that you can create and manage yourself. At the same time, it is often useful to also share the set of experiments (runs) with the ensuing benchmarking results. For legacy reasons, such sets of tasks or runs are called <code>studies</code> in the OpenML REST API. In the OpenML bindings (Python, R, Java,...) these are called either <code>sets</code> or <code>studies</code>.</p> <p>When benchmarking, you will probably use two types of sets:</p> <ul> <li>Sets of tasks. These can be created, edited, downloaded or deleted via the OpenML API. Website forms will be added soon. Also the set of underlying datasets can be easily retrieved via the API.</li> <li>Sets of runs. Likewise, these can be created, edited, downloaded or deleted via the OpenML API. On the website, these are currently simply called 'studies'. Also the set of underlying tasks, datasets and flows can be easily retrieved. It is possible to link a set of runs to a benchmark study, aimed to collect future runs on that specific set of tasks. Additional information on these will be provided in a separate page.</li> </ul>"},{"location":"benchmark/#listing-the-benchmark-suites","title":"Listing the benchmark suites","text":"<p>The current list of benchmark suites is explicitly listed on the bottom of this page. The list of all sets of tasks can also be fetched programmatically. This list includes the suite's ID (and optionally an alias), which can be used to fetch further details.</p> <p>Via the REST API, the list is returned in XML or JSON</p> REST <p>https://www.openml.org/api/v1/xml/study/list/main_entity_type/task/status/all</p> <p>Check out the API docs</p> Python example <pre><code>import openml\n\n# using the main entity type task, only benchmark suites are returned\n# each benchmark suite has an ID, some also have an alias. These can be\n# used to obtain the full details. \nstudies = openml.study.list_suites(status = 'all')\n</code></pre> Java example <pre><code>public void listBenchmarksuites() throws Exception {\n    OpenmlConnector openml = new OpenmlConnector();\n    Map&lt;String, String&gt; filters = new TreeMap&lt;String, String&gt;();\n    filters.put(\"status\", \"all\");\n    filters.put(\"main_entity_type\", \"task\");\n    filters.put(\"limit\", \"20\");\n    StudyList list = openml.studyList(filters);\n}\n</code></pre> R example <pre><code>studies = listOMLStudies()\n</code></pre>"},{"location":"benchmark/#fetching-details","title":"Fetching details","text":"<p>Using the ID or alias of a benchmark suite, you can retrieve a description and the full list of tasks and the underlying datasets.</p> <p>Via the REST API, a list of all tasks and dataset IDs is returned in XML or JSON</p> REST <p>https://www.openml.org/api/v1/xml/study/OpenML-CC18</p> <p>Check out the API docs</p> <p>In Python, the data is returned as <code>features, targets</code> numpy arrays:</p> Python example <pre><code>import openml\n\nbenchmark_suite = openml.study.get_suite('OpenML-CC18') # obtain the benchmark suite\n\nfor task_id in benchmark_suite.tasks:  # iterate over all tasks\n    task = openml.tasks.get_task(task_id)  # download the OpenML task\n    features, targets = task.get_X_and_y()  # get the data\n</code></pre> <p>In Java, the data is returned as a WEKA Instances object:</p> Java example <pre><code>public void downloadDatasets() throws Exception {\n    OpenmlConnector openml = new OpenmlConnector();\n    Study benchmarksuite = openml.studyGet(\"OpenML-CC18\", \"tasks\");\n    for (Integer taskId : benchmarksuite.getTasks()) { // iterate over all tasks\n        Task t = openml.taskGet(taskId); // download the OpenML task\n        // note that InstanceHelper is part of the OpenML-weka package\n        Instances d = InstancesHelper.getDatasetFromTask(openml, t); // obtain the dataset\n    }\n}\n</code></pre> <p>In R, the data is returned as an R dataframe:</p> R example <pre><code>library(OpenML)\ntask.ids = getOMLStudy('OpenML-CC18')$tasks$task.id # obtain the list of suggested tasks\nfor (task.id in task.ids) { # iterate over all tasks\n  task = getOMLTask(task.id) # download single OML task\n  data = as.data.frame(task) # obtain raw data set\n</code></pre>"},{"location":"benchmark/#running-and-sharing-benchmarks","title":"Running and sharing benchmarks","text":"<p>The code below demonstrates how OpenML benchmarking suites can be conveniently imported for benchmarking using the Python, Java and R APIs.</p> <p>First, the list of tasks is downloaded as already illustrated above. Next, a specific algorithm (or pipeline) can be run on each of them. The OpenML API will automatically evaluate the algorithm using the pre-set train-test splits and store the predictions and scores in a run object. This run object can then be immediately published, pushing the results to the OpenML server, so that they can be compared against all others on the same benchmark set. Uploading results requires an OpenML API key, which can be found in your account details after logging into the OpenML website.</p> REST <p>Requires POST requests: Attaching a new run to a benchmark_study Detaching a run from benchmark_study </p> Python example <pre><code>import openml\nimport sklearn\n\nopenml.config.apikey = 'FILL_IN_OPENML_API_KEY'  # set the OpenML Api Key\nbenchmark_suite = openml.study.get_suite('OpenML-CC18')  # obtain the benchmark suite\n\n# build a scikit-learn classifier\nclf = sklearn.pipeline.make_pipeline(sklearn.preprocessing.Imputer(),\n                                     sklearn.tree.DecisionTreeClassifier())\n\nfor task_id in benchmark_suite.tasks:  # iterate over all tasks\n\n    task = openml.tasks.get_task(task_id)  # download the OpenML task\n    run = openml.runs.run_model_on_task(clf, task)  # run the classifier on the task\n    score = run.get_metric_fn(sklearn.metrics.accuracy_score)  # print accuracy score\n    print('Data set: %s; Accuracy: %0.2f' % (task.get_dataset().name,score.mean()))\n    run.publish()  # publish the experiment on OpenML (optional, requires internet and an API key)\n    print('URL for run: %s/run/%d' %(openml.config.server,run.run_id))\n</code></pre> Java example <pre><code>public static void runTasksAndUpload() throws Exception {\n  OpenmlConnector openml = new OpenmlConnector();\n  openml.setApiKey(\"FILL_IN_OPENML_API_KEY\");\n  // obtain the benchmark suite\n  Study benchmarksuite = openml.studyGet(\"OpenML-CC18\", \"tasks\");\n  Classifier tree = new REPTree(); // build a Weka classifier\n  for (Integer taskId : benchmarksuite.getTasks()) { // iterate over all tasks\n    Task t = openml.taskGet(taskId); // download the OpenML task\n    Instances d = InstancesHelper.getDatasetFromTask(openml, t); // obtain the dataset\n    int runId = RunOpenmlJob.executeTask(openml, new WekaConfig(), taskId, tree);\n    Run run = openml.runGet(runId);   // retrieve the uploaded run\n  }\n}\n</code></pre> R example <pre><code>library(OpenML)\nsetOMLConfig(apikey = 'FILL_IN_OPENML_API_KEY')\nlrn = makeLearner('classif.rpart') # construct a simple CART classifier\ntask.ids = getOMLStudy('OpenML-CC18')$tasks$task.id # obtain the list of suggested tasks\nfor (task.id in task.ids) { # iterate over all tasks\n  task = getOMLTask(task.id) # download single OML task\n  data = as.data.frame(task) # obtain raw data set\n  run = runTaskMlr(task, learner = lrn) # run constructed learner\n  upload = uploadOMLRun(run) # upload and tag the run\n}\n</code></pre>"},{"location":"benchmark/#retrieving-runs-on-a-benchmarking-suites","title":"Retrieving runs on a benchmarking suites:","text":"<p>Once a benchmark suite has been created, the listing functions can be used to  obtain all results on the benchmark suite. Note that there are several other ways to select and bundle runs together. This will be featured in  a separate article on reproducible benchmarks. </p> REST (TODO) <p>https://www.openml.org/api/v1/xml/run/list/study/OpenML-CC18</p> <p>Check out the API docs</p> Python example <pre><code>benchmark_suite = openml.study.get_suite('OpenML-CC18')\nruns = openml.runs.list_runs(task=benchmark_suite.tasks, limit=1000)\n</code></pre> Java example <pre><code>public void downloadResultsBenchmarkSuite()  throws Exception {\n    Study benchmarkSuite = openml.studyGet(\"OpenML100\", \"tasks\");\n\n    Map&lt;String, List&lt;Integer&gt;&gt; filters = new TreeMap&lt;String, List&lt;Integer&gt;&gt;();\n    filters.put(\"task\", Arrays.asList(benchmarkSuite.getTasks()));\n    RunList rl = openml.runList(filters, 200, null);\n\n    assertTrue(rl.getRuns().length &gt; 0); \n}\n</code></pre> R example <pre><code>benchmark.suite = getOMLStudy(study = \"OpenML-CC18\")\nrun.ids = extractOMLStudyIds(benchmark.suite, \"run.id\")\nruns = rbindlist(lapply(run.ids, function(id) listOMLRuns(run.id = id)))\n# TODO waiting for REST API\n</code></pre>"},{"location":"benchmark/#creating-new-benchmark-suites","title":"Creating new benchmark suites","text":"<p>Additional OpenML benchmark suites can be created by defining the precise set of tasks, as well as a textual description. New datasets first need to be registered on OpenML and tasks need to be created on them.</p> <p>We have provided a GitHub repository with additional tools and scripts to build new benchmark studies, e.g. to select all datasets adhering to strict conditions, and to analyse bencharking results.</p> REST <p>Requires POST requests: Creating a benchmark suite </p> Python example <pre><code>import openml\n\n# find 250 tasks that we are interested in, e.g., the tasks that have between\n# 100 and 10000 instances and between 4 and 20 attributes\ntasks = openml.tasks.list_tasks(number_instances='100..10000', number_features='4..20', size=250)\ntask_ids = list(tasks.keys())\n\n# create the benchmark suite\n# the arguments are the alias, name, description, and list of task_ids, respectively.\nstudy = openml.study.create_benchmark_suite( \n    name=\"MidSize Suite\", \n    alias=None,\n    description=\"illustrating how to create a benchmark suite\", \n    task_ids=task_ids,\n)\nstudy_id = study.publish()\n</code></pre> Java example <pre><code>public void createBenchmarkSuite() throws Exception {\n    OpenmlConnector openml = new OpenmlConnector(\"FILL_IN_OPENML_API_KEY\");\n    // find 250 tasks that we are interested in, e.g., the tasks that have between\n    // 100 and 10000 instances and between 4 and 20 attributes\n    Map&lt;String, String&gt; filtersOrig = new TreeMap&lt;String, String&gt;();\n    filtersOrig.put(\"number_instances\", \"100..10000\");\n    filtersOrig.put(\"number_features\", \"4..20\");\n    filtersOrig.put(\"limit\", \"250\");\n    Tasks tasksOrig = client_write_test.taskList(filtersOrig);\n\n    // create the study\n    Study study = new Study(null, \"test\", \"test\", null, tasksOrig.getTaskIds(), null);\n    int studyId = openml.studyUpload(study);\n}\n</code></pre> R example <pre><code># find 250 tasks with 100 and 10000 instances and between 4 and 20 attributes\ntid = listOMLTasks(number.of.instances = c(100, 10000), number.of.features = c(4, 20), limit = 250)\nstudy = makeOMLStudy(alias = \"test_alias\", name = \"Test Upload from R\", description = \"Just testing\", task.id = tid$task.id)\nid = uploadOMLStudy(study)\n</code></pre>"},{"location":"benchmark/#updating-a-benchmark-suite","title":"Updating a benchmark suite","text":"<p>You can add tasks to a benchmark suite, or remove them.</p> REST <p>Requires POST requests: Attaching a new task Detaching a task </p> Python example <pre><code>import openml\n\n# find 250 tasks that we are interested in, e.g., the tasks that have between\n# 100 and 10000 instances and between 4 and 20 attributes\ntasks = openml.tasks.list_tasks(number_instances='100..10000', number_features='4..20', size=250)\ntask_ids = list(tasks.keys())\n\n# create the benchmark suite\nstudy = openml.study.create_benchmark_suite( \n    name=\"MidSize Suite\", \n    alias=None,\n    description=\"illustrating how to create a benchmark suite\", \n    task_ids=task_ids,\n)\nstudy_id = study.publish()\n\n# download the study from the server, for verification purposes\nstudy = openml.study.get_study(study_id)\n\n# until the benchmark suite is activated, we can also add some more tasks. Search for the letter dataset:\ntasks_new = openml.tasks.list_tasks(data_name='letter', size=1)\ntask_ids_new = list(tasks_new.keys())\nopenml.study.attach_to_study(study_id, task_ids_new)\n\n# or even remove these again\nopenml.study.detach_from_study(study_id, task_ids_new)\n\n# redownload the study\nstudy_prime = openml.study.get_study(study_id)\n\nassert(study.tasks == study_prime.tasks)\nassert(study.data == study_prime.data)\n</code></pre> Java example <pre><code>public void attachDetachStudy()  throws Exception {\n    OpenmlConnector openml = new OpenmlConnector(\"FILL_IN_OPENML_API_KEY\");\n    // find 250 tasks that we are interested in, e.g., the tasks that have between\n    // 100 and 10000 instances and between 4 and 20 attributes\n    Map&lt;String, String&gt; filtersOrig = new TreeMap&lt;String, String&gt;();\n    filtersOrig.put(\"number_instances\", \"100..10000\");\n    filtersOrig.put(\"number_features\", \"4..20\");\n    filtersOrig.put(\"limit\", \"250\");\n    Tasks tasksOrig = openml.taskList(filtersOrig);\n\n    // create the study\n    Study study = new Study(null, \"test\", \"test\", null, tasksOrig.getTaskIds(), null);\n    int studyId = openml.studyUpload(study);\n\n    // until the benchmark suite is activated, we can also add some more tasks. Search for the letter dataset:\n    Map&lt;String, String&gt; filtersAdd = new TreeMap&lt;String, String&gt;();\n    filtersAdd.put(\"data_name\", \"letter\");\n    filtersAdd.put(\"limit\", \"1\");\n    Tasks tasksAdd = openml.taskList(filtersAdd);\n    openml.studyAttach(studyId, Arrays.asList(tasksAdd.getTaskIds()));\n\n    // or even remove these again\n    openml.studyDetach(studyId, Arrays.asList(tasksAdd.getTaskIds()));\n\n    // download the study\n    Study studyDownloaded = openml.studyGet(studyId);\n    assertArrayEquals(tasksOrig.getTaskIds(), studyDownloaded.getTasks());\n}\n</code></pre> R example <pre><code>TODO\n</code></pre>"},{"location":"benchmark/#further-code-examples-and-use-cases","title":"Further code examples and use cases","text":"<p>As mentioned above, we host a GitHub repository with additional tools and scripts to easily create and use new benchmark studies. It includes:</p> <ul> <li>A Jupyter Notebook that builds a new benchmark suite with datasets that adhere to strict and complex conditions, as well as automated tests to remove tasks that are too easy for proper benchmarking.</li> <li>A Jupyter Notebook that shows how to pull in the latest state-of-the-art results for any of the benchmark suites</li> <li>A Jupyter Notebook that does a detailed analysis of all results in a benchmark suite, and an example run on the OpenML-CC18. It includes a wide range of plots and rankings to get a deeper insight into the benchmark results.</li> <li>Scripts in Python and R to facilitate common subtasks.</li> </ul> <p>We very much welcome new scripts and notebooks, or improvements to the existing ones, that help others to create benchmark suites and analyse benchmarking results.</p>"},{"location":"benchmark/#list-of-benchmarking-suites","title":"List of benchmarking suites","text":""},{"location":"benchmark/#openml-cc18","title":"OpenML-CC18","text":"<p>The OpenML-CC18 suite contains all OpenML datasets from mid-2018 that satisfy a large set of clear requirements for thorough yet practical benchmarking. It includes datasets frequently used in benchmarks published over the last years, so it can be used as a drop-in replacement for many benchmarking setups.</p> <p>List of datasets and properties</p> <p>The suite is defined as the set of all verified OpenML datasets that satisfy the following requirements:</p> <ul> <li>the number of observations are between 500 and 100000 to focus on medium-sized datasets, that are not too small and not too big,</li> <li>the number of features does not exceed 5000 features to keep the runtime of algorithms low,</li> <li>the target attribute has at least two classes</li> <li>the ratio of the minority class and the majority class is above 0.05, to eliminate highly imbalanced datasets which require special treatment for both algorithms and evaluation measures.</li> </ul> <p>We excluded datasets which:</p> <ul> <li>are artificially generated (not to confuse with simulated)</li> <li>cannot be randomized via a 10-fold cross-validation due to grouped samples or because they are time series or data streams</li> <li>are a subset of a larger dataset</li> <li>have classes with less than 20 observations</li> <li>have no source or reference available</li> <li>can be perfectly classified by a single attribute or a decision stump</li> <li>allow a decision tree to achieve 100% accuracy on a 10-fold cross-validation task</li> <li>have more than 5000 features after one-hot-encoding categorical features</li> <li>are created by binarization of regression tasks or multiclass classification tasks, or</li> <li>are sparse data (e.g., text mining data sets)</li> </ul> Detailed motivation of these decisions <p>We chose the CC18 datasets to allow for practical benchmarking based on the characteristics that might be problematic based on our experience, and to avoid common pitfalls that may invalidate benchmark studies:  </p> <ul> <li>We used at least 500 data points to allow performing cross-validation while still having a large-enough test split.</li> <li>We limited the datasets to 100.000 data points to allow the algorithms to train machine learning models in a reasonable amount of time.</li> <li>We limited the number of features to 5000 to allow the usage of algorithms which scale unfavourably in the number of features. This limitation, together with the two limitations above aims to allow running all \u201cstandard\u201d machine learning algorithms (naive bayes, linear models, support vector machines, tree-based ensemble methods and neural networks) on the benchmark suite.</li> <li>We required each dataset to have at least two classes to be able to work in a supervised classification setting.</li> <li>We require each class to have at least 20 observations to be able to perform stratified cross-validation where there is at least one observation from each class in each split. We have found that not having all classes present in all training and test sets can make several machine learning packages fail.</li> <li>We require a certain balancedness (ratio of minority class to majority class) to prevent cases where only predicting the majority class would be beneficial. This is most likely the restriction which is most debatable, but we found it very helpful to apply a large set of machine learning algorithms across several libraries to the study. We expect that future studies focus more on imbalanced datasets. </li> </ul> <p>Furthermore, we aimed to have the dataset collection as general as possible, rule out as few algorithms as possible and have it usable as easily as possible:</p> <ul> <li>We strived to remove artificial datasets as they, for example, come from textbooks and it is hard to reliably assess their difficulty. We admit that there is a blurred line between artificial and simulated datasets and do not have a perfect distinction between them (for example, a lot of phenomena can be simulated, but the outcome might be like a simple, artificial dataset). Therefore, we removed datasets if we were in doubt of whether they are simulated or artificial. </li> <li>We removed datasets which require grouped sampling because they are time series or data streams which should be treated with special care by machine learning algorithms (i.e., taking the time aspect into account). To be on the safe side, we also removed datasets where each sample constitutes a single data stream.</li> <li>We removed datasets which are a subset of larger datasets. Allowing subsets would be very subjective as there is no objective choice of a dataset subset size or a subset of the variables or classes. Therefore, creating dataset subsets would open a Pandora\u2019s Box.</li> <li>We removed datasets which have no source or reference available to potentially learn more about these datasets if we observe unexpected behavior in future studies. In contrast, we would not be able to learn more about the background of a dataset which has no description and publication attached, leaving us with a complete black box.</li> <li>We removed datasets which can be perfectly classified by a single attribute or a decision stump as they do not allow to meaningfully compare machine learning algorithms (they all achieve 100% accuracy unless the hyperparameters are set in a bogus way).</li> <li>We removed datasets where a decision tree could achieve 100% accuracy on a 10-fold cross-validation task to remove datasets which can be solved by a simple algorithm which is prone to overfitting training data. We found that this is a good indicator of too easy datasets. Obviously, other datasets will appear easy for several algorithms, and we aim to learn more about the characteristics of such datasets in future studies.</li> <li>We removed datasets which have more than 5000 features after one-hot-encoding categorical features. One-hot-encoding is the most frequent way to deal with categorical variables across the different machine learning libraries MLR, scikit-learn and WEKA. In order to limit the number of features to 5000 as explained above, we imposed the additional constraint that this should be counted after one-hot-encoding to allow wide applicability of the benchmark suite.</li> <li>We removed datasets which were created by binarization of regression tasks or multiclass classification task for similar reasons as for forbidding dataset subsets.</li> <li>We did not include sparse datasets because not all machine learning libraries (i.e., all machine learning models) can handle them gracefully, which is in contrast to our goal which is wide applicability.</li> </ul>"},{"location":"benchmark/#citing-the-openml-cc18","title":"Citing the OpenML-CC18","text":"<p>If you have used the OpenML-CC18 in a scientific publication, we would appreciate citations of core OpenML packages as well as a citation of the following paper:</p> <p>Bischl, Bernd and Casalicchio, Giuseppe and Feurer, Matthias and Hutter, Frank and Lang, Michel and Mantovani, Rafael G. and van Rijn, Jan N. and Vanschoren, Joaquin. OpenML Benchmarking Suites. arXiv 1708.0373v2 (2019): 1-6</p>"},{"location":"benchmark/#openml100","title":"OpenML100","text":"<p>The OpenML100 was a predecessor of the OpenML-CC18, consisting of 100 classification datasets. We recommend that you use the OpenML-CC18 instead, because the OpenML100 suffers from some teething issues in the design of benchmark suites. For instance, it contains several datasets that are too easy to model with today's machine learning algorithms, as well as datasets that represent time series analysis problems. These do not invalidate benchmarks run on the OpenML100, but may obfuscate the interpretation of results. The 'OpenML-CC18' handle is also more descriptive and allows easier versioning. The OpenML100 was first published in the Arxiv preprint OpenML Benchmarking Suites and the OpenML100.</p> <p>List of datasets and properties</p> <p>For reference, the OpenML100 included datasets satisfying the following requirements:</p> <ul> <li>the number of observations are between 500 and 100000 to focus on medium-sized datasets, that are not too small for proper training and not too big for practical experimentation</li> <li>the number of features does not exceed 5000 features to keep the runtime of algorithms low</li> <li>the target attribute has at least two classes</li> <li>he ratio of the minority class and the majority class is above 0.05 to eliminate highly imbalanced datasets that would obfuscate a clear analysis</li> </ul> <p>It excluded datasets which:</p> <ul> <li>cannot be randomized via a 10-fold cross-validation due to grouped samples</li> <li>have an unknown origin or no clearly defined task</li> <li>are variants of other datasets (e.g. binarized regression tasks)</li> <li>include sparse data (e.g., text mining data sets)</li> </ul>"},{"location":"benchmark/#citing-the-openml100","title":"Citing the OpenML100","text":"<p>If you have used the OpenML100 in a scientific publication, we would appreciate citations of core OpenML packages as well as a citation of the following paper:</p> <p>Bischl, Bernd and Casalicchio, Giuseppe and Feurer, Matthias and Hutter, Frank and Lang, Michel and Mantovani, Rafael G. and van Rijn, Jan N. and Vanschoren, Joaquin. OpenML Benchmarking Suites and the OpenML100. arXiv 1708.0373v1 (2017): 1-6</p>"},{"location":"benchmark/#need-help","title":"Need help?","text":"<p>We are happy to answer to any suggestion or question you may have. For general questions or issues, please open an issue in the benchmarking issue tracker. If the issue lies with one of the language-specific bindings, please post an issue in the appropriate issue tracker.</p>"},{"location":"benchmark/automl/AutoML-Benchmark/","title":"Getting Started","text":"<p>The AutoML Benchmark is a tool for benchmarking AutoML frameworks on tabular data. It automates the installation of AutoML frameworks, passing it data, and evaluating their predictions.  Our paper describes the design and showcases  results from an evaluation using the benchmark.  This guide goes over the minimum steps needed to evaluate an AutoML framework on a toy dataset.</p> <p>Full instructions can be found in the API Documentation.</p>"},{"location":"benchmark/automl/AutoML-Benchmark/#installation","title":"Installation","text":"<p>These instructions assume that Python 3.9 (or higher)  and git are installed, and are available under the alias <code>python</code> and <code>git</code>, respectively. We recommend Pyenv for managing multiple Python installations, if applicable. We support Ubuntu 22.04, but many linux and MacOS versions likely work (for MacOS, it may be necessary to have <code>brew</code> installed).</p> <p>First, clone the repository:</p> <pre><code>git clone https://github.com/openml/automlbenchmark.git --branch stable --depth 1\ncd automlbenchmark\n</code></pre> <p>Create a virtual environments to install the dependencies in:</p>"},{"location":"benchmark/automl/AutoML-Benchmark/#linux","title":"Linux","text":"<pre><code>python -m venv venv\nsource venv/bin/activate\n</code></pre>"},{"location":"benchmark/automl/AutoML-Benchmark/#macos","title":"MacOS","text":"<pre><code>python -m venv venv\nsource venv/bin/activate\n</code></pre>"},{"location":"benchmark/automl/AutoML-Benchmark/#windows","title":"Windows","text":"<pre><code>python -m venv ./venv\nvenv/Scripts/activate\n</code></pre> <p>Then install the dependencies:</p> <pre><code>python -m pip install --upgrade pip\npython -m pip install -r requirements.txt\n</code></pre> Note for Windows users <p>The automated installation of AutoML frameworks is done using shell script, which doesn't work on Windows. We recommend you use Docker to run the examples below. First, install and run <code>docker</code>.  Then, whenever there is a <code>python runbenchmark.py ...</code>  command in the tutorial, add <code>-m docker</code> to it (<code>python runbenchmark.py ... -m docker</code>).</p> Problem with the installation? <p>On some platforms, we need to ensure that requirements are installed sequentially. Use <code>xargs -L 1 python -m pip install &lt; requirements.txt</code> to do so. If problems  persist, open an issue with the error and information about your environment (OS, Python version, pip version).</p>"},{"location":"benchmark/automl/AutoML-Benchmark/#running-the-benchmark","title":"Running the Benchmark","text":"<p>To run a benchmark call the <code>runbenchmark.py</code> script specifying the framework to evaluate.</p> <p>See the API Documentation. for more information on the parameters available.</p>"},{"location":"benchmark/automl/basic_example/","title":"Random Forest Baseline","text":"<p>Let's try evaluating the <code>RandomForest</code> baseline, which uses scikit-learn's random forest:</p>"},{"location":"benchmark/automl/basic_example/#running-the-benchmark","title":"Running the Benchmark","text":""},{"location":"benchmark/automl/basic_example/#linux","title":"Linux","text":"<pre><code>python runbenchmark.py randomforest \n</code></pre>"},{"location":"benchmark/automl/basic_example/#macos","title":"MacOS","text":"<pre><code>python runbenchmark.py randomforest \n</code></pre>"},{"location":"benchmark/automl/basic_example/#windows","title":"Windows","text":"<p>As noted above, we need to install the AutoML frameworks (and baselines) in a container. Add <code>-m docker</code> to the command as shown: <pre><code>python runbenchmark.py randomforest -m docker\n</code></pre></p> <p>Important</p> <p>Future example usages will only show invocations without <code>-m docker</code> mode, but Windows users will need to run in some non-local mode.</p>"},{"location":"benchmark/automl/basic_example/#results","title":"Results","text":"<p>After running the command, there will be a lot of output to the screen that reports on what is currently happening. After a few minutes final results are shown and should  look similar to this:</p> <pre><code>Summing up scores for current run:\n               id        task  fold    framework constraint     result      metric  duration      seed\nopenml.org/t/3913         kc2     0 RandomForest       test   0.865801         auc      11.1 851722466\nopenml.org/t/3913         kc2     1 RandomForest       test   0.857143         auc       9.1 851722467\n  openml.org/t/59        iris     0 RandomForest       test  -0.120755 neg_logloss       8.7 851722466\n  openml.org/t/59        iris     1 RandomForest       test  -0.027781 neg_logloss       8.5 851722467\nopenml.org/t/2295 cholesterol     0 RandomForest       test -44.220800    neg_rmse       8.7 851722466\nopenml.org/t/2295 cholesterol     1 RandomForest       test -55.216500    neg_rmse       8.7 851722467\n</code></pre> <p>The result denotes the performance of the framework on the test data as measured by the metric listed in the metric column. The result column always denotes performance  in a way where higher is better (metrics which normally observe \"lower is better\" are converted, which can be observed from the <code>neg_</code> prefix).</p> <p>While running the command, the AutoML benchmark performed the following steps:</p> <ol> <li>Create a new virtual environment for the Random Forest experiment.      This environment can be found in <code>frameworks/randomforest/venv</code> and will be re-used      when you perform other experiments with <code>RandomForest</code>.</li> <li>It downloaded datasets from OpenML complete with a      \"task definition\" which specifies cross-validation folds.</li> <li>It evaluated <code>RandomForest</code> on each (task, fold)-combination in a separate subprocess, where:<ol> <li>The framework (<code>RandomForest</code>) is initialized.</li> <li>The training data is passed to the framework for training.</li> <li>The test data is passed to the framework to make predictions on.</li> <li>It passes the predictions back to the main process</li> </ol> </li> <li>The predictions are evaluated and reported on. They are printed to the console and      are stored in the <code>results</code> directory. There you will find:<ol> <li><code>results/results.csv</code>: a file with all results from all benchmarks conducted on your machine.</li> <li><code>results/randomforest.test.test.local.TIMESTAMP</code>: a directory with more information about the run,     such as logs, predictions, and possibly other artifacts.</li> </ol> </li> </ol> <p>Docker Mode</p> <p>When using docker mode (with <code>-m docker</code>) a docker image will be made that contains the virtual environment. Otherwise, it functions much the same way.</p>"},{"location":"benchmark/automl/basic_example/#important-parameters","title":"Important Parameters","text":"<p>As you can see from the results above, the  default behavior is to execute a short test benchmark. However, we can specify a different benchmark, provide different constraints, and even run the experiment in a container or on AWS. There are many parameters for the <code>runbenchmark.py</code> script, but the most important ones are:</p>"},{"location":"benchmark/automl/basic_example/#framework-required","title":"Framework (required)","text":"<ul> <li>The AutoML framework or baseline to evaluate and is not case-sensitive. See   integrated frameworks for a list of supported frameworks.    In the above example, this benchmarked framework <code>randomforest</code>.</li> </ul>"},{"location":"benchmark/automl/basic_example/#benchmark-optional-defaulttest","title":"Benchmark (optional, default='test')","text":"<ul> <li>The benchmark suite is the dataset or set of datasets to evaluate the framework on.   These can be defined as on OpenML as a study or task    (formatted as <code>openml/s/X</code> or <code>openml/t/Y</code> respectively) or in a local file.   The default is a short evaluation on two folds of <code>iris</code>, <code>kc2</code>, and <code>cholesterol</code>.</li> </ul>"},{"location":"benchmark/automl/basic_example/#constraints-optional-defaulttest","title":"Constraints (optional, default='test')","text":"<ul> <li> <p>The constraints applied to the benchmark as defined by default in constraints.yaml.   These include time constraints, memory constrains, the number of available cpu cores, and more.   Default constraint is <code>test</code> (2 folds for 10 min each). </p> <p>Constraints are not enforced!</p> <p>These constraints are forwarded to the AutoML framework if possible but, except for runtime constraints, are generally not enforced. It is advised when benchmarking to use an environment that mimics the given constraints.</p> Constraints can be overriden by <code>benchmark</code> <p>A benchmark definition can override constraints on a task level. This is useful if you want to define a benchmark which has different constraints for different tasks. The default \"test\" benchmark does this to limit runtime to 60 seconds instead of 600 seconds, which is useful to get quick results for its small datasets. For more information, see defining a benchmark.</p> </li> </ul>"},{"location":"benchmark/automl/basic_example/#mode-optional-defaultlocal","title":"Mode (optional, default='local')","text":"<ul> <li> <p>The benchmark can be run in four modes:</p> <ul> <li><code>local</code>: install a local virtual environment and run the benchmark on your machine.</li> <li><code>docker</code>: create a docker image with the virtual environment and run the benchmark in a container on your machine.               If a local or remote image already exists, that will be used instead. Requires Docker.</li> <li><code>singularity</code>: create a singularity image with the virtual environment and run the benchmark in a container on your machine. Requires Singularity.</li> <li><code>aws</code>: run the benchmark on AWS EC2 instances.           It is possible to run directly on the instance or have the EC2 instance run in <code>docker</code> mode.           Requires valid AWS credentials to be configured, for more information see Running on AWS.</li> </ul> </li> </ul> <p>For a full list of parameters available, run:</p> <pre><code>python runbenchmark.py --help\n</code></pre>"},{"location":"benchmark/automl/benchmark_on_openml/","title":"Example: Benchmarks on OpenML","text":"<p>In the previous examples, we used benchmarks which were defined in a local file (test.yaml and  validation.yaml, respectively).  However, we can also use tasks and benchmarking suites defined on OpenML directly from the command line. When referencing an OpenML task or suite, we can use <code>openml/t/ID</code> or <code>openml/s/ID</code> respectively as  argument for the benchmark parameter. Running on the iris task:</p> <pre><code>python runbenchmark.py randomforest openml/t/59\n</code></pre> <p>or on the entire AutoML benchmark classification suite (this will take hours!):</p> <pre><code>python runbenchmark.py randomforest openml/s/271\n</code></pre> <p>Large-scale Benchmarking</p> <p>For large scale benchmarking it is advised to parallelize your experiments, as otherwise it may take months to run the experiments. The benchmark currently only supports native parallelization in <code>aws</code> mode (by using the <code>--parallel</code> parameter), but using the <code>--task</code> and <code>--fold</code> parameters  it is easy to generate scripts that invoke individual jobs on e.g., a SLURM cluster. When you run in any parallelized fashion, it is advised to run each process on separate hardware to ensure experiments can not interfere with each other.</p>"},{"location":"benchmark/automl/important_params/","title":"Important Parameters","text":"<p>As you can see from the results above, the  default behavior is to execute a short test benchmark. However, we can specify a different benchmark, provide different constraints, and even run the experiment in a container or on AWS. There are many parameters for the <code>runbenchmark.py</code> script, but the most important ones are:</p> <p><code>Framework (required)</code></p> <ul> <li>The AutoML framework or baseline to evaluate and is not case-sensitive. See   integrated frameworks for a list of supported frameworks.    In the above example, this benchmarked framework <code>randomforest</code>.</li> </ul> <p><code>Benchmark (optional, default='test')</code></p> <ul> <li>The benchmark suite is the dataset or set of datasets to evaluate the framework on.   These can be defined as on OpenML as a study or task    (formatted as <code>openml/s/X</code> or <code>openml/t/Y</code> respectively) or in a local file.   The default is a short evaluation on two folds of <code>iris</code>, <code>kc2</code>, and <code>cholesterol</code>.</li> </ul> <p><code>Constraints (optional, default='test')</code></p> <ul> <li> <p>The constraints applied to the benchmark as defined by default in constraints.yaml.   These include time constraints, memory constrains, the number of available cpu cores, and more.   Default constraint is <code>test</code> (2 folds for 10 min each). </p> <p>Constraints are not enforced!</p> <p>These constraints are forwarded to the AutoML framework if possible but, except for runtime constraints, are generally not enforced. It is advised when benchmarking to use an environment that mimics the given constraints.</p> Constraints can be overriden by <code>benchmark</code> <p>A benchmark definition can override constraints on a task level. This is useful if you want to define a benchmark which has different constraints for different tasks. The default \"test\" benchmark does this to limit runtime to 60 seconds instead of 600 seconds, which is useful to get quick results for its small datasets. For more information, see defining a benchmark.</p> </li> </ul> <p><code>Mode (optional, default='local')</code></p> <ul> <li> <p>The benchmark can be run in four modes:</p> <ul> <li><code>local</code>: install a local virtual environment and run the benchmark on your machine.</li> <li><code>docker</code>: create a docker image with the virtual environment and run the benchmark in a container on your machine.               If a local or remote image already exists, that will be used instead. Requires Docker.</li> <li><code>singularity</code>: create a singularity image with the virtual environment and run the benchmark in a container on your machine. Requires Singularity.</li> <li><code>aws</code>: run the benchmark on AWS EC2 instances.           It is possible to run directly on the instance or have the EC2 instance run in <code>docker</code> mode.           Requires valid AWS credentials to be configured, for more information see Running on AWS.</li> </ul> </li> </ul> <p>For a full list of parameters available, run:</p> <pre><code>python runbenchmark.py --help\n</code></pre>"},{"location":"benchmark/automl/specific_task_fold_example/","title":"Example: AutoML on a specific task and fold","text":"<p>The defaults are very useful for performing a quick test, as the datasets are small and cover different task types (binary classification, multiclass classification, and  regression). We also have a \"validation\" benchmark suite for more elaborate testing that also includes missing data, categorical data,  wide data, and more. The benchmark defines 9 tasks, and evaluating two folds with a 10-minute time constraint would take roughly 3 hours (=9 tasks * 2 folds * 10 minutes, plus overhead). Let's instead use the <code>--task</code> and <code>--fold</code> parameters to run only a specific task and fold in the <code>benchmark</code> when evaluating the  flaml AutoML framework:</p> <pre><code>python runbenchmark.py flaml validation test -t eucalyptus -f 0\n</code></pre> <p>This should take about 10 minutes plus the time it takes to install <code>flaml</code>. Results should look roughly like this:</p> <pre><code>Processing results for flaml.validation.test.local.20230711T122823\nSumming up scores for current run:\n               id       task  fold framework constraint    result      metric  duration       seed\nopenml.org/t/2079 eucalyptus     0     flaml       test -0.702976 neg_logloss     611.0 1385946458\n</code></pre> <p>Similarly to the test run, you will find additional files in the <code>results</code> directory.</p>"},{"location":"concepts/","title":"Concepts","text":""},{"location":"concepts/#openml-concepts","title":"OpenML concepts","text":"<p>OpenML operates on a number of core concepts which are important to understand: </p> <p> Datasets Datasets are pretty straight-forward. Tabular datasets are self-contained, consisting of a number of rows (instances) and columns (features), including their data types. Other  modalities (e.g. images) are included via paths to files stored within the same folder. Datasets are uniformly formatted (S3 buckets with Parquet tables, JSON metadata, and media files), and are auto-converted and auto-loaded in your desired format by the APIs (e.g. in Python) in a single line of code. Example: The Iris dataset or the Plankton dataset</p> <p> Tasks A task consists of a dataset, together with a machine learning task to perform, such as classification or clustering and an evaluation method. For supervised tasks, this also specifies the target column in the data. Example: Classifying different iris species from other attributes and evaluate using 10-fold cross-validation.</p> <p> Flows A flow identifies a particular machine learning algorithm (a pipeline or untrained model) from a particular library or framework, such as scikit-learn, pyTorch, or MLR. It contains details about the structure of the model/pipeline, dependencies (e.g. the library and its version) and a list of settable hyperparameters. In short, it is a serialized description of the algorithm that in many cases can also be deserialized to reinstantiate the exact same algorithm in a particular library.  Example: scikit-learn's RandomForest or a simple TensorFlow model</p> <p> Runs A run is an experiment - it evaluates a particular flow (pipeline/model) with particular hyperparameter settings, on a particular task. Depending on the task it will include certain results, such as model evaluations (e.g. accuracies), model predictions, and other output files (e.g. the trained model). Example: Classifying Gamma rays with scikit-learn's RandomForest</p>"},{"location":"concepts/authentication/","title":"Authentication","text":"<p>OpenML is as open as possible. You can download and inspect all datasets, tasks, flows and runs through the website or the API without creating an account.</p> <p>However, if you want to upload datasets or experiments, you need to create an account, sign in, and find your API key on your profile page. This key can then be used with any of the OpenML APIs.</p>"},{"location":"concepts/authentication/#api-keys","title":"API keys","text":"<p>If you don\u2019t have an account yet, sign up now. You will receive an API key, which will authenticate you to the server and allow you to download and upload datasets, tasks, runs and flows.</p> <ul> <li>Create an OpenML account (free) on https://www.openml.org.</li> <li>After logging in, open your profile page. Click on the avatar on the top right, and choose 'Your Profile'.</li> <li>Click on 'API key' to find your API key. You can also reset it if needed.</li> </ul> <p>To store your API key locally (to permanently authenticate), create a plain text file ~/.openml/config with the line 'apikey=MYKEY', replacing MYKEY with your API key. The config file must be in the directory ~/.openml/config and exist prior to importing the openml module.</p>"},{"location":"concepts/benchmarking/","title":"Collections and benchmarks","text":"<p>You can combine tasks and runs into collections, to run experiments across many tasks at once and collect all results. Each collection gets its own page, which can be linked to publications so that others can find all the details online.</p>"},{"location":"concepts/benchmarking/#benchmarking-suites","title":"Benchmarking suites","text":"<p>Collections of tasks can be published as benchmarking suites. Seamlessly integrated into the OpenML platform, benchmark suites standardize the setup, execution, analysis, and reporting of benchmarks. Moreover, they make benchmarking a whole lot easier: - all datasets are uniformly formatted in standardized data formats - they can be easily downloaded programmatically through APIs and client libraries - they come with machine-readable meta-information, such as the occurrence of missing  values, to train algorithms correctly - standardized train-test splits are provided to ensure that results can be objectively compared - results can be shared in a reproducible way through the APIs - results from other users can be easily downloaded and reused </p> <p>You can search for all existing benchmarking suites or create your own. For all further details, see the benchmarking guide.</p> <p></p>"},{"location":"concepts/benchmarking/#benchmark-studies","title":"Benchmark studies","text":"<p>Collections of runs can be published as benchmarking studies. They contain the results of all runs (possibly millions) executed on a specific benchmarking suite. OpenML allows you to easily download all such results at once via the APIs, but also visualized them online in the Analysis tab (next to the complete list of included tasks and runs). Below is an example of a benchmark study for AutoML algorithms.</p> <p></p>"},{"location":"concepts/data/","title":"Data","text":""},{"location":"concepts/data/#discovery","title":"Discovery","text":"<p>OpenML allows fine-grained search over thousands of machine learning datasets. Via the website, you can filter by many dataset properties, such as size, type, format, and many more. Via the APIs you have access to many more filters, and you can download a complete table with statistics of all datasest. Via the APIs you can also load datasets directly into your preferred data structures such as numpy (example in Python). We are also working on better organization of all datasets by topic </p> <p></p>"},{"location":"concepts/data/#sharing","title":"Sharing","text":"<p>You can upload and download datasets through the website or though the APIs (recommended). You can share data directly from common data science libraries, e.g. from Python or R dataframes, in a few lines of code. The OpenML APIs will automatically extract lots of meta-data and store all datasets in a uniform format.</p> <pre><code>    import pandas as pd\n    import openml as oml\n\n    # Create an OpenML dataset from a pandas dataframe\n    df = pd.DataFrame(data, columns=attribute_names)\n    my_data = oml.datasets.functions.create_dataset(\n        name=\"covertype\", description=\"Predicting forest cover ...\",\n        licence=\"CC0\", data=df\n    )\n\n    # Share the dataset on OpenML\n    my_data.publish()\n</code></pre> <p>Every dataset gets a dedicated page on OpenML with all known information, and can be edited further online.</p> <p></p> <p>Data hosted elsewhere can be referenced by URL. We are also working on interconnecting OpenML with other machine learning data set repositories </p>"},{"location":"concepts/data/#automated-analysis","title":"Automated analysis","text":"<p>OpenML will automatically analyze the data and compute a range of data quality characteristics. These include simple statistics such as the number of examples and features, but also potential quality issues (e.g. missing values) and more advanced statistics (e.g. the mutual information in the features and benchmark performances of simple models). These can be useful to find, filter and compare datasets, or to automate data preprocessing.  We are also working on simple metrics and automated dataset quality reports </p> <p>The Analysis tab (see image below, or try it live) also shows an automated and interactive analysis of all datasets. This runs on open-source Python code via Dash and we welcome all contributions </p> <p></p> <p>The third tab, 'Tasks', lists all tasks created on the dataset. More on that below.</p>"},{"location":"concepts/data/#dataset-id-and-versions","title":"Dataset ID and versions","text":"<p>A dataset can be uniquely identified by its dataset ID, which is shown on the website and returned by the API. It's <code>1596</code> in the <code>covertype</code> example above. They can also be referenced by name and ID. OpenML assigns incremental version numbers per upload with the same name. You can also add a free-form <code>version_label</code> with every upload.</p>"},{"location":"concepts/data/#dataset-status","title":"Dataset status","text":"<p>When you upload a dataset, it will be marked <code>in_preparation</code> until it is (automatically) verified. Once approved, the dataset will become <code>active</code> (or <code>verified</code>). If a severe issue has been found with a dataset, it can become <code>deactivated</code> (or <code>deprecated</code>) signaling that it should not be used. By default, dataset search only returns verified datasets, but you can access and download datasets with any status.</p>"},{"location":"concepts/data/#special-attributes","title":"Special attributes","text":"<p>Machine learning datasets often have special attributes that require special handling in order to build useful models. OpenML marks these as special attributes.</p> <p>A <code>target</code> attribute is the column that is to be predicted, also known as dependent variable. Datasets can have a default target attribute set by the author, but OpenML tasks can also overrule this. Example: The default target variable for the MNIST dataset is to predict the class from pixel values, and most supervised tasks will have the class as their target. However, one can also create a task aimed at predicting the value of pixel257 given all the other pixel values and the class column.</p> <p><code>Row id</code> attributes indicate externally defined row IDs (e.g. <code>instance</code> in dataset 164). <code>Ignore</code> attributes are other columns that should not be included in training data (e.g. <code>Player</code> in dataset 185). OpenML will clearly mark these, and will (by default) drop these columns when constructing training sets.</p>"},{"location":"concepts/flows/","title":"Flows","text":"<p>Flows are machine learning pipelines, models, or scripts. They are typically uploaded directly from machine learning libraries (e.g. scikit-learn, pyTorch, TensorFlow, MLR, WEKA,...) via the corresponding APIs. Associated code (e.g., on GitHub) can be referenced by URL.</p>"},{"location":"concepts/flows/#analysing-algorithm-performance","title":"Analysing algorithm performance","text":"<p>Every flow gets a dedicated page with all known information. The Analysis tab shows an automated interactive analysis of all collected results. For instance, below are the results of a scikit-learn pipeline including missing value imputation, feature encoding, and a RandomForest model. It shows the results across multiple tasks, and how the AUC score is affected by certain hyperparameters.</p> <p></p> <p>This helps to better understand specific models, as well as their strengths and weaknesses.</p>"},{"location":"concepts/flows/#automated-sharing","title":"Automated sharing","text":"<p>When you evaluate algorithms and share the results, OpenML will automatically extract all the details of the algorithm (dependencies, structure, and all hyperparameters), and upload them in the background.</p> <pre><code>    from sklearn import ensemble\n    from openml import tasks, runs\n\n    # Build any model you like.\n    clf = ensemble.RandomForestClassifier()\n\n    # Evaluate the model on a task\n    run = runs.run_model_on_task(clf, task)\n\n    # Share the results, including the flow and all its details.\n    run.publish()\n</code></pre>"},{"location":"concepts/flows/#reproducing-algorithms-and-experiments","title":"Reproducing algorithms and experiments","text":"<p>Given an OpenML run, the exact same algorithm or model, with exactly the same hyperparameters, can be reconstructed within the same machine learning library to easily reproduce earlier results. </p> <pre><code>    from openml import runs\n\n    # Rebuild the (scikit-learn) pipeline from run 9864498\n    model = openml.runs.initialize_model_from_run(9864498)\n</code></pre> <p>Note</p> <p>You may need the exact same library version to reconstruct flows. The API will always state the required version. We aim to add support for VMs so that flows can be easily (re)run in any environment </p>"},{"location":"concepts/runs/","title":"Runs","text":""},{"location":"concepts/runs/#automated-reproducible-evaluations","title":"Automated reproducible evaluations","text":"<p>Runs are experiments (benchmarks) evaluating a specific flows on a specific task. As shown above, they are typically submitted automatically by machine learning libraries through the OpenML APIs), including lots of automatically extracted meta-data, to create reproducible experiments. With a few for-loops you can easily run (and share) millions of experiments.</p>"},{"location":"concepts/runs/#online-organization","title":"Online organization","text":"<p>OpenML organizes all runs online, linked to the underlying data, flows, parameter settings, people, and other details. See the many examples above, where every dot in the scatterplots is a single OpenML run.</p>"},{"location":"concepts/runs/#independent-server-side-evaluation","title":"Independent (server-side) evaluation","text":"<p>OpenML runs include all information needed to independently evaluate models. For most tasks, this includes all predictions, for all train-test splits, for all instances in the dataset, including all class confidences. When a run is uploaded, OpenML automatically evaluates every run using a wide array of evaluation metrics. This makes them directly comparable with all other runs shared on OpenML. For completeness, OpenML will also upload locally computed evaluation metrics and runtimes. </p> <p>New metrics can also be added to OpenML's evaluation engine, and computed for all runs afterwards. Or, you can download OpenML runs and analyse the results any way you like.</p> <p>Note</p> <p>Please note that while OpenML tries to maximise reproducibility, exactly reproducing all results may not always be possible because of changes in numeric libraries,  operating systems, and hardware.</p>"},{"location":"concepts/sharing/","title":"Sharing (under construction)","text":"<p>Currently, anything on OpenML can be shared publicly or kept private to a single user. We are working on sharing features that allow you to share your materials with other users without making them entirely public. Watch this space </p>"},{"location":"concepts/tagging/","title":"Tagging","text":"<p>Datasets, tasks, runs and flows can be assigned tags, either via the web interface or the API. These tags can be used to search and annotate datasets, or simply to better organize your own datasets and experiments.</p> <p>For example, the tag OpenML-CC18 refers to all tasks included in the OpenML-CC18 benchmarkign suite.</p>"},{"location":"concepts/tasks/","title":"Tasks","text":"<p>Tasks describe what to do with the data. OpenML covers several task types, such as classification and clustering. Tasks are containers including the data and other information such as train/test splits, and define what needs to be returned. They are machine-readable so that you can automate machine learning experiments, and easily compare algorithms evaluations (using the exact same train-test splits) against all other benchmarks shared by others on OpenML.</p>"},{"location":"concepts/tasks/#collaborative-benchmarks","title":"Collaborative benchmarks","text":"<p>Tasks are real-time, collaborative benchmarks (e.g. see MNIST below). In the Analysis tab, you can view timelines and leaderboards, and learn from all prior submissions to design even better algorithms.</p> <p></p>"},{"location":"concepts/tasks/#discover-the-best-algorithms","title":"Discover the best algorithms","text":"<p>All algorithms evaluated on the same task (with the same train-test splits) can be directly compared to each other, so you can easily look up which algorithms perform best overall, and download their exact configurations. Likewise, you can look up the best algorithms for similar tasks to know what to try first.</p> <p></p>"},{"location":"concepts/tasks/#automating-benchmarks","title":"Automating benchmarks","text":"<p>You can search and download existing tasks, evaluate your algorithms, and automatically share the results (which are stored in a run). Here's what this looks like in the Python API. You can do the same across hundreds of tasks at once.</p> <pre><code>    from sklearn import ensemble\n    from openml import tasks, runs\n\n    # Build any model you like\n    clf = ensemble.RandomForestClassifier()\n\n    # Download any OpenML task (includes the datasets)\n    task = tasks.get_task(3954)\n\n    # Automatically evaluate your model on the task\n    run = runs.run_model_on_task(clf, task)\n\n    # Share the results on OpenML.\n    run.publish()\n</code></pre> <p>You can create new tasks via the website or via the APIs as well.</p>"},{"location":"contributing/","title":"How to Contribute","text":"<p>OpenML is an open source project, hosted on GitHub. We welcome everybody to help improve OpenML, and make it more useful for everyone.</p> <p>Mission</p> <p>We want to make machine learning open and accessible for the benefit of all of humanity. OpenML offers an entirely open online platform for machine learning datasets, models, and experiments, making them easy to use and share to facilitate global collaboration and extensive automation.</p>"},{"location":"contributing/#want-to-get-involved","title":"Want to get involved?","text":"<p>Awesome, we're happy to have you! </p>"},{"location":"contributing/#who-are-we","title":"Who are we?","text":"<p>We are a group of friendly people who are excited about open science and machine learning. </p> <p>Read more about who we are, what we stand for, and how to get in touch.</p>"},{"location":"contributing/#we-need-help","title":"We need help!","text":"<p>We are currently looking for help with:</p> <p> User feedback (best via GitHub issues, but email or Slack is also fine)</p> <ul> <li>Frontend / UX / Design of the website</li> <li>Backend / API</li> <li>Outreach / making OpenML better known (especially in non-ML-communities, where people have data but no analysis experise)</li> <li>Helping with the interfaces (Python,R,Julia,Java) and tool integrations</li> <li>Helping with documenting the interfaces or the API</li> <li>What could we do better to get new users started? Help us to figure out what is difficult to understand about OpenML. If you are a new user, you are the perfect person for this!</li> </ul>"},{"location":"contributing/#beginner-issues","title":"Beginner issues","text":"<p>Check out the issues labeled Good first issue or help wanted (you need to be logged into GitHub to see these)</p>"},{"location":"contributing/#change-the-world","title":"Change the world","text":"<p>If you have your own ideas on how you want to contribute, please get in touch! We are very friendly and open to new ideas </p>"},{"location":"contributing/#communication-channels","title":"Communication channels:","text":"<p>We have several communication channels set up for different purposes:</p>"},{"location":"contributing/#github","title":"GitHub","text":"<p>https://github.com/openml</p> <ul> <li>Issues (members and users can complain)</li> <li>Request new features</li> </ul> <p>Anyone with a GitHub account can write issues. We are happy if people get involved by writing issues, so don't be shy </p> <p>Please post issues in the relevant issue tracker.</p> <ul> <li> OpenML Core - Web services and API</li> <li> Website - The (new) OpenML website</li> <li> Docs - The documentation pages</li> <li> Python API - The Python API</li> <li> R API - The OpenML R package</li> <li> Java API - The Java API and Java-based plugins</li> <li> Datasets - For issues about datasets</li> <li> Blog - The OpenML Blog</li> </ul>"},{"location":"contributing/#slack","title":"Slack","text":"<p>https://openml.slack.com</p> <ul> <li>Informal communication</li> </ul> <p>We use slack for day to day discussions and news. If you want to join the OpenML slack chat, please message us (openmlHQ@googlegroups.com).</p>"},{"location":"contributing/#twitter-open_ml","title":"Twitter (@open_ml)","text":"<p>https://twitter.com/open_ml</p> <ul> <li>News</li> <li>Publicly relevant information</li> </ul>"},{"location":"contributing/#blog","title":"Blog","text":"<p>https://blog.openml.org</p> <ul> <li>Tutorials</li> <li>News</li> <li>Open discussions</li> </ul>"},{"location":"contributing/#contributors-bot","title":"Contributors bot","text":"<p>We use all contributors bot to add contributors to the repository README. You can check how to use this here. You can contribute in a lot of ways including code, blogs, content, design and talks. You can find the emoji key here .</p>"},{"location":"contributing/OpenML-Docs/","title":"Documentation","text":""},{"location":"contributing/OpenML-Docs/#general-documentation","title":"General Documentation","text":"<p>High-quality and up-to-date documentation are crucial. If you notice any mistake in these documentation pages, click the  button (on the top right). It will open up an editing page on GitHub (you do need to be logged in). When you are done, add a small message explaining the change and click 'commit changes'. On the next page, just launch the pull request. We will then review it and approve the changes, or discuss them if necessary.</p> <p>The sources are generated by MkDocs, using the Material theme. Check these docs to see what is possible in terms of styling.</p> <p>OpenML is a big project with multiple repositories. To keep the documentation close to the code, it will always be kept in the relevant repositories (see below), and  combined into these documentation pages using MkDocs multirepo.</p> <p>Developer note</p> <p>To work on the documentation locally, do the following: <pre><code>git clone https://github.com/openml/docs.git\npip install -r requirements.txt\n</code></pre> To build the documentation, run <code>mkdocs serve</code> in the top directory (with the <code>mkdocs.yml</code> file). Any changes made after that will be hot-loaded.</p> <p>The documentation will be auto-deployed with every push or merge with the master branch of <code>https://www.github.com/openml/docs/</code>. In the background, a CI job will run <code>mkdocs gh-deploy</code>, which will build the HTML files and push them to the gh-pages branch of openml/docs. <code>https://docs.openml.org</code> is just a reverse proxy for <code>https://openml.github.io/docs/</code>.</p>"},{"location":"contributing/OpenML-Docs/#python-api","title":"Python API","text":"<p>To edit the tutorial, you have to edit the <code>reStructuredText</code> files on openml-python/doc. When done, you can do a pull request.</p> <p>To edit the documentation of the python functions, edit the docstrings in the Python code. When done, you can do a pull request.</p> <p>Developer note</p> <p>A CircleCI job will automatically render the documentation on every GitHub commit, using Sphinx. For inclusion in these documentation pages, it will also be rendered in markdown and imported.</p>"},{"location":"contributing/OpenML-Docs/#r-api","title":"R API","text":"<p>To edit the tutorial, you have to edit the <code>Rmarkdown</code> files on openml-r/vignettes.</p> <p>To edit the documentation of the R functions, edit the Roxygen documention next to the functions in the R code.</p> <p>Developer note</p> <p>A Travis job will automatically render the documentation on every GitHub commit, using knitr. The Roxygen documentation is updated every time a new version is released on CRAN.</p>"},{"location":"contributing/OpenML-Docs/#java-api","title":"Java API","text":"<p>The Java Tutorial is written in markdown and can be edited the usual way (see above).</p> <p>To edit the documentation of the Java functions, edit the documentation next to the functions in the Java code.</p> <ul> <li>Javadocs: https://www.openml.org/docs/</li> </ul> <p>Developer note</p> <p>A Travis job will automatically render the documentation on every GitHub commit, using Javadoc.</p>"},{"location":"contributing/OpenML-Docs/#rest-api","title":"REST API","text":"<p>The REST API is documented using Swagger.io, in YAML. This generates a nice web interface that also allows trying out the API calls using your own API key (when you are logged in).</p> <p>You can edit the sources on SwaggerHub. When you are done, export to json and replace the downloads/swagger.json file in the OpenML main GitHub repository. You need to do a pull request that is then reviewed by us. When we merge the new file the changes are immediately available.</p> <p>The data API can be edited in the same way.</p>"},{"location":"contributing/Style/","title":"Style guide","text":"<p>These are some (non-mandatory) style guidelines to make the OpenML experience more pleasant and consistent for everyone.</p>"},{"location":"contributing/Style/#logos","title":"Logos","text":"<p> (SVG)</p>"},{"location":"contributing/Style/#colors","title":"Colors","text":"<p>We use the Material Design color system, and especially the colors green[400], yellow[800], blue[800], red[400], green[400], yellow[800], pink[400], and purple[400].</p> <p>Primary colors are #1E88E5 (general), #000482 (dark), and #b5b7ff (light).</p>"},{"location":"contributing/resources/","title":"Resources","text":""},{"location":"contributing/resources/#resources","title":"Resources","text":""},{"location":"contributing/resources/#database-snapshots","title":"Database snapshots","text":"<p>Everything uploaded to OpenML is available to the community. The nightly snapshot of the public database contains all experiment runs, evaluations and links to datasets, implementations and result files. In SQL format (gzipped). You can also download the Database schema.</p> <p> Nightly database SNAPSHOT</p> <p>If you want to work on the website locally, you'll also need the schema for the 'private' database with non-public information.</p> <p> Private database schema</p>"},{"location":"contributing/resources/#legacy-resources","title":"Legacy Resources","text":"<p>OpenML is always evolving, but we keep hosting the resources that were used in prior publications so that others may still build on them.</p> <p> The experiment database used in Vanschoren et al. (2012) Experiment databases. Machine Learning 87(2), pp 127-158. You'll need to import this database (we used MySQL) to run queries. The database structure is described in the paper. Note that most of the experiments in this database have been rerun using OpenML, using newer algorithm implementations and stored in much more detail.</p> <p> The Expos\u00e9 ontology used in the same paper, and described in more detail here and here. Expos\u00e9 is used in designing our databases, and we aim to use it to export all OpenML data as Linked Open Data.</p>"},{"location":"contributing/resources/#other-dataset-repositories","title":"Other dataset repositories","text":"<p>We keep a list of other dataset repositories all over the world</p>"},{"location":"contributing/backend/API-development/","title":"Code structure","text":"<p>Phasing out</p> <p>This documentation is about the old PHP-based API, which wil be phased out in favor of (e.g. the new Python-based API (using FastAPI)). See the 'Server' tab for more information.</p>"},{"location":"contributing/backend/API-development/#important-resources","title":"Important resources","text":"<p>REST API docs: www.openml.org/apis</p> <p>Controller: https://github.com/openml/OpenML/blob/master/openml_OS/controllers/Api_new.php</p> <p>Models: https://github.com/openml/OpenML/tree/master/openml_OS/models/api/v1</p> <p>Templates: https://github.com/openml/OpenML/tree/master/openml_OS/views/pages/api_new/v1</p>"},{"location":"contributing/backend/API-development/#golden-rules-for-development","title":"Golden Rules for Development","text":"<ol> <li>Code Maintainability before anything else. The code has to be understandable, and if not conflicting with that, short. Avoid code duplications as much as possible.</li> <li>The API controller is the only entity giving access to the API models. Therefore, the responsibility for API access can be handled by the controller</li> <li>Read-Only operations are of the type GET. Operations that make changes in the database are of type POST or DELETE. Important, because this is the way the controller determines to allow users with a given set of privileges to access functions.</li> <li>Try to avoid direct queries to the database. Instead, use the respective models functions: 'get()', 'getWhere()', 'getById()', insert(), etc (Please make yourself familiar with the basic model: read-only and write)</li> <li>No external program/script execution during API calls (with one exception: data split generation). This makes the API unnecessarily slow, hard to debug and vulnerable to crashes. If necessary, make a cronjob that executes the program / script</li> </ol>"},{"location":"contributing/backend/API-development/#backend-code-structure","title":"Backend code structure","text":"<p>The high-level architecture of the website, including the controllers for different parts of the website (REST API, html, ...) and connections to the database.</p>"},{"location":"contributing/backend/API-development/#code","title":"Code","text":"<p>The source code is available in the 'OpenML' repository: https://github.com/openml/OpenML </p>"},{"location":"contributing/backend/API-development/#important-files-and-folders","title":"Important files and folders","text":"<p>In this section we go through all important files and folder of the system.</p>"},{"location":"contributing/backend/API-development/#root-directory","title":"Root directory","text":"<p>The root directory of OpenML contains the following files and folders.</p> <ul> <li> <p>system: This folder contains all files provided by   CodeIgniter 2.1.3. The contents of this folder is   beyond the scope of this document, and not relevant for extending   OpenML. All the files in this folder are in the same state as they   were provided by Ellislabs, and none of these files should ever be   changed.</p> </li> <li> <p>sparks: Sparks is a package management system for   Codeigniter that allows for instant installation of libraries into   the application. This folder contains two libraries provided by   third party software developers, oauth1 (based on version 1 the   oauth protocol) and oauth2 (similarly, based on version 2 of the   oauth protocol). The exact contents of this folder is beyond the   scope of this document and not relevant for extending OpenML.</p> </li> <li> <p>openml_OS: All files in this folder are written specifically   for OpenML. When extending the functionality OpenML, usually one of   the files in this folder needs to be adjusted. As a thorough   understanding of the contents of this folder is vital for extending   OpenML, we will discuss the contents of this folder in   [[URL Mapping]] in more detail.</p> </li> <li> <p>index.php: This is the \u201cbootstrap\u201d file of the system.   Basically, every page request on OpenML goes through this file (with   the css, images and javascript files as only exception). It then   determines which CodeIgniter and OpenML files need to be included.   This file should not be edited.</p> </li> <li> <p>.htaccess: This file (which configures the Apache Rewrite   Engine) makes sure that all URL requests will be directed to   <code>index.php</code>. Without this file, we would need to include <code>index.php</code>   explicitly in every URL request. This file makes sure that all other   URL requests without <code>index.php</code> embedded in it automatically will   be transformed to <code>index.php</code>. Eg.,   http://www.openml.org/frontend/page/home will be rewritten to   http://www.openml.org/index.php/frontend/page/home. This will be   explained in detail in [[URL Mapping]].</p> </li> <li> <p>css: A folder containing all stylesheets. These are important   for the layout of OpenML.</p> </li> <li> <p>data: A folder containing data files, e.g., datasets,   implementation files, uploaded content. Please note that this folder   does not necessarily needs to be present in the root directory. The   OpenML Base Config file determines the   exact location of this folder.</p> </li> <li> <p>downloads: Another data folder, containing files like the most   recent database snapshot.</p> </li> <li> <p>img: A folder containing all static images shown on the webpage.</p> </li> <li> <p>js: A folder containing all used Javascript files and libraries,   including third party libraries like jQuery and datatables.</p> </li> <li> <p>Various other files, like .gitignore, favicon.ico, etc.</p> </li> </ul>"},{"location":"contributing/backend/API-development/#openml_os","title":"openml_OS","text":"<p>This folder is (in CodeIgniter jargon) the \u201cApplication folder\u201d, and contains all files relevant to OpenML. Within this folder, the following folders should be present: (And also some other folders, but these are not used by OpenML)</p> <ul> <li> <p>config: A folder containing all config files. Most notably, it   contains the file BASE_CONFIG.php, in which all system   specific variables are set; the config items within this file   differs over various installations (e.g., on localhost,   <code>openml.org</code>). Most other config files, like   database.php, will receive their values from   BASE_CONFIG.php. Other important config files are   autoload.php, determining which CodeIgniter / OpenML   files will be loaded on any request, openML.php,   containing config items specific to OpenML, and   routes.php, which will be explained in   [[URL Mapping]].</p> </li> <li> <p>controllers: In the Model/View/Controller design pattern, all   user interaction goes through controllers. In a webapplication   setting this means that every time a URL gets requested, exactly one   controller gets invoked. The exact dynamics of this will be   explained in [[URL Mapping]].</p> </li> <li> <p>core: A folder that contains CodeIgniter specific files. These   are not relevant for the understanding of OpenML.</p> </li> <li> <p>helpers: This folder contains many convenience functions.   Wikipedia states: \u201cA convenience function is a non-essential   subroutine in a programming library or framework which is intended   to ease commonly performed tasks\u201d. For example the   file_upload_helper.php contains many functions that   assist with uploading of files. Please note that a helper function   must be explicitly loaded in either the autoload config or the files   that uses its functions.</p> </li> <li> <p>libraries: Similar to sparks, this folder contains libraries   specifically written for CodeIgniter. For example, the library used   for all user management routines is in this folder.</p> </li> <li> <p>models: In the Model/View/Controller design pattern, models   represent the state of the system. In a webapplication setting, you   could say that a model is the link to the database. In OpenML,   almost all tables of the database are represented by a model. Each   model has general functionality applicable to all models (e.g.,   retrieve all records, retrieve record with constraints, insert   record) and functionality specific to that model (e.g., retrieve a   dataset that has certain data properties). Most models extend an   (abstract) base class, located in the abstract folder.   This way, all general functionality is programmed and maintained in   one place.</p> </li> <li> <p>third_party: Although the name might suggests differently, this   folder contains all OpenML Java libraries.</p> </li> <li> <p>views: In the Model/View/Controller design pattern, the views   are the way information is presented on the screen. In a   webapplication setting, a view usually is a block of (PHP generated)   HTML code. The most notable view is frontend_main.php,   which is the template file determining the main look and feel of   OpenML. Every single page also has its own specific view (which is   parsed within frontend_main.php). These pages can be   found (categorized by controller and name) in the pages   folder. More about this structure is explained in   [[URL Mapping]].</p> </li> </ul>"},{"location":"contributing/backend/API-development/#frontend-code-structure","title":"Frontend code structure","text":"<p>Architecture and libraries involved in generating the frontend functions.</p> <p>Code: https://github.com/openml/website/tree/master/openml_OS/views</p>"},{"location":"contributing/backend/API-development/#high-level","title":"High-level","text":"<p>All pages are generated by first loading frontend_main.php. This creates the 'shell' in which the content is loaded. It loads all css and javascript libraries, and contains the html for displaying headers and footers.</p>"},{"location":"contributing/backend/API-development/#create-new-page","title":"Create new page","text":"<p>The preferred method is creating a new folder into the folder <code>&lt;root_directory&gt;/openml_OS/views/pages/frontend</code> This page can be requested by <code>http://www.openml.org/frontend/page/&lt;folder_name&gt;</code> or just <code>http://www.openml.org/&lt;folder_name&gt;</code> This method is preferred for human readable webpages, where the internal actions are simple, and the output is complex. We will describe the files that can be in this folder.</p> <ul> <li> <p>pre.php: Mandatory file. Will be executed first. Do not make   this file produce any output! Can be used to pre-render data, or set   some variables that are used in other files.</p> </li> <li> <p>body.php: Highly recommended file. Intended for displaying the   main content of this file. Will be rendered at the right location   within the template file (<code>frontend_main.php</code>).</p> </li> <li> <p>javascript.php: Non-mandatory file. Intended for javascript   function on which <code>body.php</code> relies. Will be rendered within a   javascript block in the header of the page.</p> </li> <li> <p>post.php: Non mandatory file. Will only be executed when a POST   request is done (e.g., when a HTML form was send using the POST   protocol). Will be executed after <code>pre.php</code>, but before the   rendering process (and thus, before <code>body.php</code> and   <code>javascript.php</code>). Should handle the posted input, e.g., file   uploads.</p> </li> </ul> <p>It is also recommended to add the newly created folder to the mapping in the <code>routes.php</code> config file. This way it can also be requested by the shortened version of the URL. (Note that we deliberately avoided to auto-load all pages into this file using a directory scan, as this makes the webplatform slow.)</p>"},{"location":"contributing/backend/API-development/#url-to-page-mapping","title":"URL to Page Mapping","text":"<p>Most pages in OpenML are represented by a folder in /openml_OS/views/pages/frontend The contents of this folder will be parsed in the template <code>frontend_main.php</code> template, as described in [[backend]]. In this section we explain the way an URL is mapped to a certain OpenML page."},{"location":"contributing/backend/API-development/#url-anatomy","title":"URL Anatomy","text":"<p>By default, CodeIgniter (and OpenML) accepts a URL in the following form: <code>http://www.openml.org/index.php/&lt;controller&gt;/&lt;function&gt;/&lt;p1&gt;/&lt;pN&gt;/&lt;free&gt;</code> The various parts in the URL are divided by slashes. Every URL starts with the protocol and server name (in the case of OpenML this is <code>http://www.openml.org/</code>). This is followed by the bootstrap file, which is always the same, i.e., <code>index.php</code>. The next part indicates the controller that needs to be invoked; typically this is <code>frontend</code>, <code>rest_api</code> or <code>data</code>, but it can be any file from the <code>openml_OS</code> folder <code>controllers</code>. Note that the suffix <code>.php</code> should not be included in the URL.</p> <p>The next part indicates which function of the controller should be invoked. This should be a existing, public function from the controller that is indicated in the controller part. These functions might have one or more parameters that need to be set. This is the following part of the URL (indicated by <code>p1</code> and <code>pN</code>). The parameters can be followed by anything in free format. Typically, this free format is used to pass on additional parameters in <code>name</code> - <code>value</code> format, or just a way of adding a human readable string to the URL for SEO purposes.</p> <p>For example, the following URL <code>http://www.openml.org/index.php/frontend/page/home</code> invokes the function <code>page</code> from the <code>frontend</code> controller and sets the only parameter of this function, <code>$indicator</code>, to value <code>home</code>. The function <code>page</code> loads the content of the specified folder (<code>$indicator</code>) into the main template. In this sense, the function <code>page</code> can be seen as some sort of specialized page loader.</p>"},{"location":"contributing/backend/API-development/#url-shortening","title":"URL Shortening","text":"<p>Since it is good practice to have URL\u2019s as short as possible, we have introduced some logic that shortens the URL\u2019s. Most importantly, the URL part that invokes <code>index.php</code> can be removed at no cost, since this file is always invoked. For this, we use Apache\u2019s rewrite engine. Rules for rewriting URL\u2019s can be found in the <code>.htaccess</code> file, but is suffices to say that any URL in the following format <code>http://www.openml.org/index.php/&lt;controller&gt;/&lt;function&gt;/&lt;params&gt;</code> can due to the rewrite engine also be requested with <code>http://www.openml.org/&lt;controller&gt;/&lt;function&gt;/&lt;params&gt;</code></p> <p>Furthermore, since most of the pages are invoked by the function <code>page</code> of the <code>frontend</code> controller (hence, they come with the suffix <code>frontend/page/page_name</code>) we also created a mapping that maps URL\u2019s in the following form <code>http://www.openml.org/&lt;page_name&gt;</code> to <code>http://www.openml.org/frontend/page/&lt;page_name&gt;</code> Note that Apache\u2019s rewrite engine will also add <code>index.php</code> to this. The exact mapping can be found in <code>routes.php</code> config file.</p>"},{"location":"contributing/backend/API-development/#additional-mappings","title":"Additional Mappings","text":"<p>Additionally, a mapping is created from the following type of URL: <code>http://www.openml.org/api/&lt;any_query_string&gt;</code> to <code>http://www.openml.org/rest_api/&lt;any_query_string&gt;</code> This was done for backwards compatibility. Many plugins make calls to the not-existing <code>api</code> controller, which are automatically redirected to the <code>rest_api</code> controller.</p>"},{"location":"contributing/backend/API-development/#exceptions","title":"Exceptions","text":"<p>It is important to note that not all pages do have a specific page folder. The page folders are a good way of structuring complex GUI\u2019s that need to be presented to the user, but in cases where the internal state changes are more important than the GUI\u2019s, it might be preferable to make the controller function print the output directly. This happens for example in the functions of <code>rest_api.php</code> and <code>free_query.php</code> (although the former still has some files in the views folder that it refers to).</p>"},{"location":"contributing/backend/API-development/#xsd-schemas","title":"XSD Schemas","text":"<p>In order to ensure data integrity on the server, data that passed to upload functions is checked against XSD schema's. This ensures that the data that is uploaded is in the correct format, and does not contain any illegal characters. XSD schema's can be obtained through the API (exact links are provided in the API docs, but for example: https://www.openml.org/api/v1/xsd/openml.data.upload (where openml.data.upload can be replaced by any other schema's name). Also XML examples are provided, e.g., https://www.openml.org/api/v1/xml_example/data . The XSD schema's are exactly the same as used on the server. Whenever an upload fails and the server mentions an XML/XSD verification error, please run the uploaded xml against one of the provided XSD schema's, for example on this webtool: http://www.freeformatter.com/xml-validator-xsd.html</p> <p>In order to maintain one XSD schema for both uploading and downloading stuff, the XSD sometimes contains more fields than seem necessary from the offset. Usually, the additional fields that are indicated as such in the comments (for example, in the upload dataset xsd this are the id, upload_date, etc fields). The XSD's maintain basically three consistencies functions:</p> <ul> <li>Ensure that the correct fields are uploaded</li> <li>Ensure that the fields contain the correct data types.</li> <li>Ensure that the fields do not contain to much characters for the database to upload.</li> </ul> <p>For the latter two, it is important to note that the XSD seldom accept default string content (i.e., xs:string). Rather, we use self defined data types, that use regular expressions to ensure the right content. Examples of these are oml:system_string128, oml:casual_string128, oml:basic_latin128, where the oml prefix is used, the name indicates the level of restriction and the number indicates the maximum size of the field.</p> <p>IMPORTANT: The maximum field sizes are (often) chosen with great care. Do not extend them without consulting other team members.</p>"},{"location":"contributing/backend/API-development/#user-authentication","title":"User authentication","text":"<p>Authentication towards the server goes by means of a so-called api_key (a hexa-decimal string which uniquely identifies a user). Upon interaction with the server, the client passes this api_key to the server, and the server checks the rights of the user. Currently this goes by means of a get or post variable, but in the future we might want to use a header field (because of security). It is recommended to refresh your api_key every month.</p> <p>IMPORTANT: Most authentication operations are handled by the ION_Auth library (http://benedmunds.com/ion_auth/). DO NOT alter information directly in the user table, always use the ION_Auth API.</p> <p>A user can be part of one or many groups. The following user groups exists:</p> <ol> <li>Admin Group: With great power comes great responsibility. Admin users can overrule all security checks on the server, i.e., delete a dataset or run that is not theirs, or even delete a flow that contains runs.</li> <li>Normal Group: Level that is required for read/write interaction with the server. Almost all users are part of this group.</li> <li>Read-only Group: Level that can be used for read interaction with the server. If a user is part of this group, but not part of 'Normal Group', he is allowed to download content, but can not upload or delete content.</li> <li>Backend Group: (Work in Progress) Level that has more privileges than 'Normal Group'. Can submit Data Qualities and Evaluations.</li> </ol> <p>The ION_Auth functions in_group(), add_to_group(), remove_from_group() and get_users_groups() are key towards interaction with these tables.</p>"},{"location":"contributing/backend/Datasets/","title":"Datasets","text":""},{"location":"contributing/backend/Datasets/#data-formats","title":"Data Formats","text":"<p>OpenML aims to achieve full data interoperability, meaning that you can load all datasets in a uniform way (a 'universal dataloader'). This requires that all datasets are stored in the same dataformat (or a set of interoperable formats), or at least have a version of it stored in that format. After an intensive study, which you can read on our blog, we settled on the Parquet format.</p> <p>This means that all OpenML datasets can be retrieved in the Parquet format. They are also stored on our servers in this format. Oftentimes, you will not notice this, as the OpenML clients can automatically convert data into your preferred data structures, and be fed directly into machine learning workflows. For example:</p> <pre><code>import openml\ndataset = openml.datasets.get_dataset(\"Fashion-MNIST\")     # Returns the dataset meta-data \nX, y, _, _ = dataset.get_data(dataset_format=\"dataframe\",  # Downloads the data and returns a Pandas dataframe\n                target=dataset.default_target_attribute)\n\nfrom sklearn.ensemble import GradientBoostingClassifier         # Using a sklearn model as an example\nmodel = GradientBoostingClassifier(n_estimators=10).fit(X, y)   # Set hyperparameters and train the model \n</code></pre> <p>To guarantee interoperability, we focus on a limited set of data formats.</p>"},{"location":"contributing/backend/Datasets/#tabular-data","title":"Tabular data","text":"<p>OpenML has historically focussed on tabular data, and has extensive support for all kinds of tabular data. As explained above, we store all data in the Parquet format. You can upload data from many different data structures, such as Pandas dataframes and R dataframes, after which they will be converted and stored in Parquet. You can also upload datasets as CSV files or ARFF files, and we aim to allow direct Parquet uploads soon.</p> <p>ARFF legacy</p> <p>At the moment, some aspects of OpenML still has a dependency on the ARFF format. This will be fully phased out in favor of Parquet.</p>"},{"location":"contributing/backend/Datasets/#image-data","title":"Image data","text":"<p>OpenML generally supports other data types by requiring a 'header table', a table (stored in Parquet) listing all data instances with additional meta-data (e.g. classes, bounding boxes,...) and references to data files, such as images (e.g. JPGs), stored in seperate folders. See our blog post for details. We will provide more detailed guidelines here as soon as possible.</p>"},{"location":"contributing/backend/Datasets/#data-repositories","title":"Data repositories","text":"<p>This is a list of public dataset repositories that offer additional useful machine learning datasets. These have widely varying data formats, so they require manual selection, parsing and meta-data extraction.</p> <p>A collection of sources made by different users</p> <ul> <li>https://github.com/caesar0301/awesome-public-datasets</li> <li>https://dreamtolearn.com/ryan/1001_datasets</li> <li>https://en.wikipedia.org/wiki/List_of_datasets_for_machine-learning_research</li> <li>https://pathmind.com/wiki/open-datasets</li> <li>https://paperswithcode.com/</li> <li>https://medium.com/towards-artificial-intelligence/best-datasets-for-machine-learning-data-science-computer-vision-nlp-ai-c9541058cf4f</li> <li>https://lionbridge.ai/datasets/the-50-best-free-datasets-for-machine-learning/</li> <li>https://www.v7labs.com/open-datasets?utm_source=v7&amp;utm_medium=email&amp;utm_campaign=edu_outreach</li> </ul> <p>Machine learning dataset repositories (mostly already in OpenML)</p> <ul> <li>UCI: https://archive.ics.uci.edu/ml/index.html</li> <li>KEEL: http://sci2s.ugr.es/keel/datasets.php</li> <li>LIBSVM: http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/</li> <li>AutoWEKA datasets: http://www.cs.ubc.ca/labs/beta/Projects/autoweka/datasets/</li> <li>skData package: https://github.com/jaberg/skdata/tree/master/skdata</li> <li>Rdatasets: http://vincentarelbundock.github.io/Rdatasets/datasets.html</li> <li>DataBrewer: https://github.com/rmax/databrewer</li> <li>liac-arff: https://github.com/renatopp/arff-datasets</li> </ul> <p>MS Open datasets:</p> <ul> <li>https://azure.microsoft.com/en-us/services/open-datasets/catalog/</li> </ul> <p>APIs (mostly defunct):</p> <ul> <li>databrewer (Python): https://pypi.org/project/databrewer/</li> <li>PyDataset (Python): https://github.com/iamaziz/PyDataset (wrapper for Rdatasets?)</li> <li>RDatasets (R): https://github.com/vincentarelbundock/Rdatasets</li> </ul> <p>Time series / Geo data:</p> <ul> <li>Data commons: https://datacommons.org/</li> <li>UCR: http://timeseriesclassification.com/</li> <li>Older version: http://www.cs.ucr.edu/~eamonn/time_series_data/</li> </ul> <p>Deep learning datasets (mostly image data)</p> <ul> <li>https://www.tensorflow.org/datasets/catalog/overview</li> <li>http://deeplearning.net/datasets/</li> <li>https://deeplearning4j.org/opendata</li> <li>http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html</li> <li>https://paperswithcode.com/datasets</li> </ul> <p>Extreme classification:</p> <ul> <li>http://manikvarma.org/downloads/XC/XMLRepository.html</li> </ul> <p>MLData (down)</p> <ul> <li>http://mldata.org/</li> </ul> <p>AutoWEKA datasets:</p> <ul> <li>http://www.cs.ubc.ca/labs/beta/Projects/autoweka/datasets/</li> </ul> <p>Kaggle public datasets</p> <ul> <li>https://www.kaggle.com/datasets</li> </ul> <p>RAMP Challenge datasets</p> <ul> <li>http://www.ramp.studio/data_domains</li> </ul> <p>Wolfram data repository</p> <ul> <li>http://datarepository.wolframcloud.com/</li> </ul> <p>Data.world</p> <ul> <li>https://data.world/</li> </ul> <p>Figshare (needs digging, lots of Excel files)</p> <ul> <li>https://figshare.com/search?q=dataset&amp;quick=1</li> </ul> <p>KDNuggets list of data sets (meta-list, lots of stuff here):</p> <ul> <li>http://www.kdnuggets.com/datasets/index.html</li> </ul> <p>Benchmark Data Sets for Highly Imbalanced Binary Classification</p> <ul> <li>http://www.cs.gsu.edu/~zding/research/imbalance-data/x19data.txt</li> </ul> <p>Feature Selection Challenge Datasets</p> <ul> <li>http://www.nipsfsc.ecs.soton.ac.uk/datasets/</li> <li>http://featureselection.asu.edu/datasets.php</li> </ul> <p>BigML's list of 1000+ data sources</p> <ul> <li>http://blog.bigml.com/2013/02/28/data-data-data-thousands-of-public-data-sources/</li> </ul> <p>Massive list from Data Science Central.</p> <ul> <li>http://www.datasciencecentral.com/profiles/blogs/data-sources-for-cool-data-science-projects</li> </ul> <p>R packages (also see https://github.com/openml/openml-r/issues/185)</p> <ul> <li>http://stat.ethz.ch/R-manual/R-patched/library/datasets/html/00Index.html</li> <li>mlbench</li> <li>Stata datasets: http://www.stata-press.com/data/r13/r.html</li> </ul> <p>UTwente Activity recognition datasets:</p> <ul> <li>http://ps.ewi.utwente.nl/Datasets.php</li> </ul> <p>Vanderbilt:</p> <ul> <li>http://biostat.mc.vanderbilt.edu/wiki/Main/DataSets</li> </ul> <p>Quandl</p> <ul> <li>https://www.quandl.com</li> </ul> <p>Microarray data:</p> <ul> <li>http://genomics-pubs.princeton.edu/oncology/</li> <li>http://svitsrv25.epfl.ch/R-doc/library/multtest/html/golub.html</li> </ul> <p>Medical data:</p> <ul> <li>http://www.healthdata.gov/</li> <li>http://homepages.inf.ed.ac.uk/rbf/IAPR/researchers/PPRPAGES/pprdat.htm</li> <li>http://hcup-us.ahrq.gov/</li> <li>https://www.cms.gov/Research-Statistics-Data-and-Systems/Statistics-Trends-and-Reports/Medicare-Provider-Charge-Data/Physician-and-Other-Supplier.html</li> <li>https://nsduhweb.rti.org/respweb/homepage.cfm</li> <li>http://orwh.od.nih.gov/resources/policyreports/womenofcolor.asp</li> </ul> <p>Nature.com Scientific data repositories list</p> <ul> <li>https://www.nature.com/sdata/policies/repositories</li> </ul>"},{"location":"contributing/backend/Java-App/","title":"Evaluation Engine","text":"<p>Phasing out</p> <p>This documentation is about the older Java-based version of the OpenML evaluation engine, which will be phased out. These parts are being rewritten as a set of independent services in Python.</p> <p>When you submit datasets or experiments (runs) to OpenML, they will be processed by set of server-side processes, combined in the 'Evaluation Engine':</p> <ul> <li>It extracts the features in tabular datasets and their statistical types</li> <li>It computes a set of dataset characteristics (meta-features), e.g. the number of features and classes, that help with search and filtering, or to compute dataset similarity measures</li> <li>It evaluates experiments using a set of server-side evaluation metrics that are computed uniformly for all experiments so that they are comparable</li> <li>It creates consistent train-test splits based on task characteristics.</li> </ul> <p>The application that implements the evaluation engine was originally implemented in Java because it bulds on the Weka API. It is invoked from the OpenML API by means of a CLI interface. Typically, a call looks like this:</p> <p><code>java -jar webapplication.jar -config \"api_key=S3CR3T_AP1_K3Y\" -f evaluate_run -r 500</code></p> <p>Which in this case executes the webapplication jar, invokes the function \"evaluate run\" and gives it parameter run id 500. The config parameter can be used to set some config items, in this case the api_key is mandatory. Every OpenML user has an api_key, which can be downloaded from their OpenML profile page. The response of this function is a call to the OpenML API uploading evaluation results to the OpenML database. Note that in this case the PHP website invokes the Java webapplication, which makes a call to the PHP website again, albeit another endpoint. </p> <p>The webapplication does not have direct writing rights into the database. All communication to the database goes by means of the OpenML Connector, which communicates with the OpenML API. As a consequence, the webapplication could run on any system, i.e., there is no formal need for the webapplication to be on the same server as the website code. This is important, since this created modularity, and not all servers provide a command line interface to PHP scripts.</p> <p>Another example is the following:</p> <p><code>java -jar webapplication -config \"api_key=S3CR3T_AP1_K3Y\" -f all_wrong -r 81,161 -t 59</code></p> <p>Which takes a comma separated list of run ids (no spaces) and a task id as input and outputs the test examples on the dataset on which all algorithms used in the runs produced wrong examples (in this case, weka.BayesNet_K2 and weka.SMO, respectively). An error will be displayed if there are runs not consistent with the task id in there. </p>"},{"location":"contributing/backend/Java-App/#extending-the-java-app","title":"Extending the Java App","text":"<p>The bootstrap class of the webapplication is</p> <p><code>org.openml.webapplication.Main</code></p> <p>It automatically checks authentication settings (such as api_key) and the determines which function to invoke. </p> <p>It uses a switch-like if - else contruction to facilitate the functionalities of the various functions. Additional functions can be added to this freely. From there on, it is easy to add functionality to the webapplication. </p> <p>Parameters are handled using the Apache Commons CommandLineParser class, which makes sure that the passed parameters are available to the program. </p> <p>In order to make new functionalities available to the website, there also needs to be programmed an interface to the function, somewhere in the website. The next section details on that. </p>"},{"location":"contributing/backend/Java-App/#interfacing-from-the-openml-api","title":"Interfacing from the OpenML API","text":"<p>By design, the REST API is not allowed to communicate with the Java App. All interfaces with the Java webapplication should go through other controllers of the PHP CodeIgniter framework., for example api_splits. Currently, the website features two main API's. These are represented by a Controller. Controllers can be found in the folder openml_OS/controllers. Here we see: * api_new.php, representing the REST API * api_splits.php, representing an API interfacing to the Java webapplication. </p>"},{"location":"contributing/backend/Java-App/#helper-functions","title":"Helper functions","text":"<p>The Java code is available in the 'OpenML' repository: https://github.com/openml/OpenML/tree/master/Java</p>"},{"location":"contributing/backend/Java-App/#components","title":"Components","text":"<p>Support for tasks:  </p> <ul> <li>foldgeneration: Java code for generating cross-validation folds. Can be used from command line.</li> <li>splitgeneration: Split generator for cross validation and holdout. Unsure what's the difference with the previous?</li> <li>generate_predictions: Helper class to build prediction files based on WEKA output. Move to WEKA repository?</li> <li>evaluate_predictions: The evaluation engine computing evaluation scores based on submitted predictions</li> </ul>"},{"location":"contributing/backend/Local-Installation/","title":"Local Installation","text":"<p>Test server</p> <p>OpenML has a fully functional test server accessible at <code>test.openml.org</code> that you can use to develop against. For many cases, this is sufficient for development, and a full local installation is not required.</p> <p>Backend evolution</p> <p>OpenML has grown organically, since before the current ecosystem of python tools for platform building. We are currently rewriting the entire backend using state-of-the-art Python tools (e.g. FastAPI) so that the entire platform can be easily installed locally in one go. We plan this to be available early/mid 2025. Please get in touch  if you want to know more or want to contribute.</p>"},{"location":"contributing/backend/Local-Installation/#using-docker-compose","title":"Using Docker Compose","text":"<p>The easiest way to set up a local version of OpenML is to use Docker Compose following the instructions here (thanks to Jos van der Velde!): https://github.com/openml/services.</p> <p>If you run into problems, please post an issue in the same github repo.</p>"},{"location":"contributing/backend/Local-Installation/#installation-from-scratch","title":"Installation from scratch","text":"<p>If you want to install a local version of OpenML from scratch please follow the steps mentioned below. Note that this does not include the Kubernetes and S3 Object storage components that we use in production.</p>"},{"location":"contributing/backend/Local-Installation/#requirements","title":"Requirements","text":"<p>You'll need to have the following software running: * Apache Webserver, (with the rewrite module enabled. Is installed by default, not enabled.) * MySQL Server. * PHP 5.5 or higher (comes also with Apache) Or just a XAMP (Mac), LAMP (Linux) or WAMP (Windows) package, which conveniently contains all these applications.</p>"},{"location":"contributing/backend/Local-Installation/#databases","title":"Databases","text":"<p>Next, OpenML runs on two databases, a public database with all experiment information, and a private database, with information like user accounts etc. The latest version of both databases can be downloaded here: https://docs.openml.org/resources</p> <p>Obviously, the private database does not include any actual user account info.</p>"},{"location":"contributing/backend/Local-Installation/#backend","title":"Backend","text":"<p>The source code is available in the 'OpenML' repository: https://github.com/openml/OpenML</p> <p>OpenML is written in PHP, and can be 'installed' by copying all files in the 'www' or 'public_html' directory of Apache.</p> <p>After that, you need to provide your local paths and database accounts and passwords using the config file in: 'APACHE_WWW_DIR'/openml_OS/config/BASE_CONFIG.php.</p> <p>If everything is configured correctly, OpenML should now be running.</p>"},{"location":"contributing/backend/Local-Installation/#search-indices","title":"Search Indices","text":"<p>If you want to run your own (separate) OpenML instance, and store your own data, you'll also want to build your own search indices to show all data on the website. The OpenML website is based on the ElasticSearch stack. To install it, follow the instructions here: http://knowm.org/how-to-set-up-the-elk-stack-elasticsearch-logstash-and-kibana/</p>"},{"location":"contributing/backend/Local-Installation/#initialization","title":"Initialization","text":"<p>This script wipes all OpenML server data and rebuilds the database and search index. Replace 'openmldir' with the directory where you want OpenML to store files.</p> <pre><code># delete data from server\nsudo rm -rf /openmldir/*\nmkdir /openmldir/log\n\n# delete database\nmysqladmin -u \"root\" -p\"yourpassword\" DROP openml_expdb\nmysql -h localhost -u root -p\"yourpassword\" -e \"TRUNCATE openml.file;\"\n\n# reset ES search index\necho \"Deleting and recreating the ES index: \"\ncurl -XDELETE http://localhost:9200/openml\ncurl -XPUT 'localhost:9200/openml?pretty' -H 'Content-Type: application/json' -d'\n{\n    \"settings\" : {\n        \"index\" : {\n            \"number_of_shards\" : 3,\n            \"number_of_replicas\" : 2\n        }\n    }\n}\n'\n\n# go to directory with the website source code\ncd /var/www/openml.org/public_html/\n\n# reinitiate the database\nmysql -u root -p\"yourpassword!\" &lt; downloads/openml_expdb.sql\n\n# fill important columns\nsudo php index.php cron install_database\n\n# rebuild search index\nsudo php index.php cron initialize_es_indices\nsudo php index.php cron build_es_indices\n\nsudo chown apache:apache /openmldir/log\nsudo chown apache:apache /openmldir/log/*\n</code></pre>"},{"location":"contributing/clients/Client-API-Standards/","title":"Client development","text":""},{"location":"contributing/clients/Client-API-Standards/#building-clients","title":"Building clients","text":"<p>You can access OpenML datasets, pipelines, benchmarks, and much more, through a range of client APIs. Well-developed clients exist in Python, R, Java, and several other languages. Please see their documentation (in the other tabs) for more guidance of how to contribute to them.</p> <p>If you want to develop your own client (e.g. for a new language), please check out the following resources:  </p> <ul> <li>REST API: all endpoints to GET, POST, or DELETE resources</li> <li>Metadata Standard: how we describe datasets and all other OpenML resources</li> <li>Minimal standards (below) for uniform client configuration and caching mechanisms, to make the client behavior more uniform across languages.</li> </ul> <p>Integrating tools</p> <p>If you want to integrate OpenML into machine learning and data science tools, it's often easier to build on one of the existing clients,  which often can be used as is or extended. For instance, see how to extend the Python API to integrate OpenML into Python tools. </p>"},{"location":"contributing/clients/Client-API-Standards/#minimal-standards","title":"Minimal standards","text":""},{"location":"contributing/clients/Client-API-Standards/#configuration-file","title":"Configuration file","text":"<p>The configuration file resides in a directory <code>.openml</code> in the home directory of the user and is called config. It consists of <code>key = value</code> pairs which are seperated by newlines. The following keys are defined:</p> <ul> <li>apikey:<ul> <li>required to access the server</li> </ul> </li> <li>server:<ul> <li>default: <code>http://www.openml.org</code></li> </ul> </li> <li>verbosity:<ul> <li>0: normal output</li> <li>1: info output</li> <li>2: debug output</li> </ul> </li> <li>cachedir:<ul> <li>if not given, will default to <code>file.path(tempdir(), \"cache\")</code>.</li> </ul> </li> </ul>"},{"location":"contributing/clients/Client-API-Standards/#caching","title":"Caching","text":""},{"location":"contributing/clients/Client-API-Standards/#cache-invalidation","title":"Cache invalidation","text":"<p>All parts of the entities which affect experiments are immutable. The entities dataset and task have a flag <code>status</code> which tells the user whether they can be used safely.</p>"},{"location":"contributing/clients/Client-API-Standards/#file-structure","title":"File structure","text":"<p>Caching should be implemented for</p> <ul> <li>datasets</li> <li>tasks</li> <li>splits</li> <li>predictions</li> </ul> <p>and further entities might follow in the future. The cache directory <code>$cache</code> should be specified by the user when invoking the API. The structure in the cache directory should be as following:</p> <ul> <li>One directory for the following entities:<ul> <li><code>$cache/datasets</code></li> <li><code>$cache/tasks</code></li> <li><code>$cache/runs</code></li> </ul> </li> <li>For every dataset there is an extra directory for which the name is the dataset ID, e.g. <code>$cache/datasets/2</code> for the dataset with OpenML ID 2.<ul> <li>The dataset should be called <code>dataset.pq</code> or <code>dataset.arff</code></li> <li>Every other file should be named by the API call which was used to obtain it. The XML returned by invoking <code>openml.data.qualities</code> should therefore be called qualities.xml.</li> </ul> </li> <li>For every task there is an extra directory for which the name is the task ID, e.g. <code>$cache/tasks/1</code><ul> <li>The task file should be called <code>task.xml</code>.</li> <li>The splits accompanying a task are stored in a file <code>datasplits.arff</code>.</li> </ul> </li> <li>For every run there is an extra directory for which the name is the run ID, e.g. <code>$cache/run/1</code><ul> <li>The predictions should be called <code>predictions.arff</code>.</li> </ul> </li> </ul>"},{"location":"contributing/clients/Rest/","title":"REST API","text":"<p>OpenML offers a RESTful Web API, with predictive URLs, for uploading and downloading machine learning resources. Try the REST API Documentation to see examples of all calls, and test them right in your browser.</p>"},{"location":"contributing/clients/Rest/#getting-started","title":"Getting started","text":"<ul> <li>REST services can be called using simple HTTP GET or POST actions.</li> <li>The REST Endpoint URL is <code>https://www.openml.org/api/v1/</code></li> <li>The default endpoint returns data in XML. If you prefer JSON, use the endpoint <code>https://www.openml.org/api/v1/json/</code>. Note that, to upload content, you still need to use XML (at least for now).</li> </ul>"},{"location":"contributing/clients/Rest/#testing","title":"Testing","text":"<p>For continuous integration and testing purposes, we have a test server offering the same API, but which does not affect the production server.</p> <ul> <li>The test server REST Endpoint URL is <code>https://test.openml.org/api/v1/</code></li> </ul>"},{"location":"contributing/clients/Rest/#error-messages","title":"Error messages","text":"<p>Error messages will look like this:</p> <pre><code>&lt;oml:error xmlns:oml=\"http://openml.org/error\"&gt;\n&lt;oml:code&gt;100&lt;/oml:code&gt;\n&lt;oml:message&gt;Please invoke legal function&lt;/oml:message&gt;\n&lt;oml:additional_information&gt;Additional information, not always available.&lt;/oml:additional_information&gt;\n&lt;/oml:error&gt;\n</code></pre> <p>All error messages are listed in the API documentation. E.g. try to get a non-existing dataset:</p> <ul> <li>in XML: https://www.openml.org/api_new/v1/data/99999</li> <li>in JSON: https://www.openml.org/api_new/v1/json/data/99999</li> </ul>"},{"location":"contributing/clients/Rest/#examples","title":"Examples","text":"<p>You need to be logged in for these examples to work.</p>"},{"location":"contributing/clients/Rest/#download-a-dataset","title":"Download a dataset","text":"<ul> <li>User asks for a dataset using the /data/{id} service. The <code>dataset id</code> is typically part of a task, or can be found on OpenML.org.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting the dataset. This can be OpenML, but also any other data repository. Try it now</li> </ul>"},{"location":"contributing/clients/Rest/#download-a-flow","title":"Download a flow","text":"<ul> <li>User asks for a flow using the /flow/{id} service and a <code>flow id</code>. The <code>flow id</code> can be found on OpenML.org.</li> <li>OpenML returns a description of the flow as an XML file (or JSON). Try it now</li> <li>The flow description contains the URL where the flow can be downloaded (e.g. GitHub), either as source, binary or both, as well as additional information on history, dependencies and licence. The user calls the right URL to download it.</li> <li>The flow is returned by the server hosting it. This can be OpenML, but also any other code repository. Try it now</li> </ul>"},{"location":"contributing/clients/Rest/#download-a-task","title":"Download a task","text":"<ul> <li>User asks for a task using the /task/{id} service and a <code>task id</code>. The <code>task id</code> is typically returned when searching for tasks.</li> <li>OpenML returns a description of the task as an XML file (or JSON). Try it now</li> <li>The task description contains the <code>dataset id</code>(s) of the datasets involved in this task. The user asks for the dataset using the /data/{id} service and the <code>dataset id</code>.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting it. This can be OpenML, but also any other data repository. Try it now</li> <li>The task description may also contain links to other resources, such as the train-test splits to be used in cross-validation. The user calls that URL to download the train-test splits.</li> <li>The train-test splits are returned by OpenML. Try it now</li> </ul>"},{"location":"contributing/clients/metadata_definition/","title":"Metadata definition","text":"<p>OpenML is at its core a meta-database, from which datasets, pipelines (flows), experiments (runs) and other entities can be downloaded and uploaded, all described using a clearly defined meta-data standard. In this document, we describe the standard how to upload entities to OpenML and what the resulting database state will be.</p> <p> Croissant</p> <p>OpenML has partnered with MLCommons, Google, Kaggle, HuggingFace, and a consortium of other partners to define a new metadata standard for machine learning datasets:  Croissant! You can already download all OpenML datasets in the Croissant format, and we're working further supporting and extending Croissant.</p> <p>Below is the OpenML metadata standard for version 1 of the API.</p>"},{"location":"contributing/clients/metadata_definition/#data","title":"Data","text":"<p>Data is uploaded through the function post data. The following files are needed:</p> <ul> <li><code>description</code>: An XML adhiring to the XSD schema.</li> <li><code>dataset</code>: An ARFF file containing the data (optional, if not set, there should be an URL in the description, pointing to this file).   Uploading any other files will result in an error.</li> </ul>"},{"location":"contributing/clients/metadata_definition/#tasks","title":"Tasks","text":"<p>Tasks are uploaded through the function post task. The following files are needed:</p> <ul> <li><code>description</code>: An XML adhering to the XSD schema.   Uploading any other files will result in an error.</li> </ul> <p>The task file should contain several input fields. These are a name and value combination of fields that are marked to be relevant by the task type definition. There are several task type definitions, e.g.:</p> <ul> <li>Supervised Classification</li> <li>Supervised Regression</li> <li>Learning Curve</li> <li>Data Stream Classification</li> </ul> <p>Note that the task types themselves are flexible content (ideally users can contribute task types) and therefore the documents are not part of the OpenML definition. The task types define which input fields should be set, when creating a task.</p> <p>Duplicate tasks (i.e., same value for <code>task_type_id</code> and all <code>input</code> fields equal) will be rejected.</p> <p>When creating a task, the API checks for all of the input fields whether the input is legitimate. (Todo: describe the checks and what they depend on).</p>"},{"location":"contributing/clients/metadata_definition/#flow","title":"Flow","text":"<p>Flows are uploaded through the function post flow. The following file is needed:</p> <ul> <li><code>description</code>: An XML adhering to the XSD schema.   Uploading any other files will result in an error.</li> </ul> <p>Duplicate flows (i.e., same values for <code>name</code> and <code>external_version</code>) will be rejected.</p>"},{"location":"contributing/clients/metadata_definition/#runs","title":"Runs","text":"<p>Runs are uploaded through the function post run. The following files are needed:</p> <ul> <li><code>description</code>: An XML adhering to the XSD schema.</li> <li><code>predictions</code>: An ARFF file containing the predictions (optional, depending on the task).</li> <li><code>trace</code>: An ARFF file containing the run trace (optional, depending on the flow).   Uploading any other files will result in an error.</li> </ul>"},{"location":"contributing/clients/metadata_definition/#predictions","title":"Predictions","text":"<p>The contents of the prediction file depends on the task type.</p>"},{"location":"contributing/clients/metadata_definition/#task-type-supervised-classification","title":"Task type: Supervised classification","text":"<p>Example predictions file</p> <ul> <li>repeat NUMERIC</li> <li>fold NUMERIC</li> <li>row_id NUMERIC</li> <li>confidence.{$classname}: optional. various columns, describing the confidence per class. The values of these columns should add to 1 (precision 1e-6).</li> <li>(proposal) decision_function.{$classname}: optional. various columns, describing decision function per class.</li> <li>prediction {$classname}   Runs that have a different set of columns will be rejected.</li> </ul>"},{"location":"contributing/clients/metadata_definition/#trace","title":"Trace","text":"<p>Example trace file</p> <ul> <li>repeat: cross-validation repeat</li> <li>fold: cross-validation fold</li> <li>iteration: the index order within this repeat/fold combination</li> <li>evaluation (float): the evaluation score that was attached based on the validation set</li> <li>selected {True, False}: Whether in this repeat/run combination this was the selected hyperparameter configuration (exactly one should be tagged with True)</li> <li>Per optimized parameter a column that has the name of the parameter and the prefix \"parameter_\"</li> <li>setup_string: Due to legacy reasons accepted, but will be ignored by the default evaluation engine</li> </ul> <p>Traces that have a different set of columns will be rejected.</p>"},{"location":"contributing/website/Dash/","title":"Dash visualization","text":"<p>Dash is a python framework which is suitable for building data visualization dashboards using pure python. Dash is written on top of plotly, react and flask and the graphs are defined using plotly python. The dash application is composed of two major parts :</p> <ul> <li><code>Layout</code> - Describes how the dashboard looks like</li> <li><code>Callbacks</code> - Used to update graphs, tables in the layout and makes the dashboard interactive.</li> </ul>"},{"location":"contributing/website/Dash/#files","title":"Files","text":"<p>The dash application is organized as follows:</p> <ul> <li> <p><code>dashapp.py</code></p> </li> <li> <p>Creates the dash application</p> </li> <li>The dash app is embedded in the flask app passed to <code>create_dash_app</code> function</li> <li> <p>This file need not be modified to create a new plot</p> </li> <li> <p><code>layouts.py</code></p> </li> <li> <p>contains the layout for all the pages</p> </li> <li><code>get_layout_from_data</code>- returns layout of data visualization</li> <li><code>get_layout_from_task</code>- returns layout of taskvisualization</li> <li><code>get_layout_from_flow</code>- returns layout of flow visualization</li> <li><code>get_layout_from_run</code> - returns layout of run visualization</li> <li> <p>This file needs to be modified to add a new plot (data, task, flow, run)</p> </li> <li> <p><code>callbacks.py</code></p> </li> <li>Registers all the callbacks for the dash application</li> <li>This file needs to be modified to add a new plot, especially if the plot needs to be interactive</li> </ul>"},{"location":"contributing/website/Dash/#how-the-dashboard-works","title":"How the dashboard works","text":"<p>In this dash application, we need to create the layout of the page dynamically based on the entered URL. For example, [http://127.0.0.1:5000/dashboard/data/5] needs to return the layout for dataset id #5 whereas [http://127.0.0.1:5000/dashboard/run/5] needs to return the layout for run id #5.</p> <p>Hence , the dash app is initially created with a dummy <code>app.layout</code> by dashapp.py and the callbacks are registered for the app using <code>register_callbacks</code> function.</p> <ul> <li> <p>render_layout is the callback which dynamically renders layout. Once the dash app is running, the first callback which is fired is <code>render_layout.</code>   This is the main callback invoked when a URL with a data , task, run or flow ID is entered.   Based on the information in the URL, this method returns the layout.</p> </li> <li> <p>Based on the URL, get_layout_from_data, get_layout_from_task, get_layout_from_flow, get_layout_from_run are called.   These functions define the layout of the page - tables, html Divs, tabs, graphs etc.</p> </li> <li> <p>The callbacks corresponding to each component in the layout are invoked to update the components dynamically and   make the graphs interactive. For example, update_scatter_plot in <code>data_callbacks.py</code> updates the scatter plot   component in the data visualization dashboard.</p> </li> </ul>"},{"location":"contributing/website/Flask/","title":"Flask backend","text":"<p>We use Flask as our web framework. It handles user authentication, dataset upload, task creation, and other aspects that require server-side interaction. It is designed to be independent from the OpenML API. This means that you can use it to create your own personal frontend for OpenML, using the main OpenML server to provide the data. Of course, you can also link it to your own local OpenML setup.</p>"},{"location":"contributing/website/Flask/#design","title":"Design","text":"<p>Out flask app follows Application factories design pattern. A new app instance can be created by: <pre><code>    from autoapp import create_app\n    app = create_app(config_object)\n</code></pre></p> <p>The backend is designed in a modular fashion with flask Blueprints. Currently, the flask app consists of two blueprints public and user:</p> <ul><li>Public blueprint: contains routes that do not require user authentication or authorization. like signup and forgot password.</li> <li>User blueprint: Contains routes which require user authentication like login, changes in profile and fetching API key.</li></ul> <p>New blueprints can be registered in `server/app.py` with register_blueprints function:</p> <pre><code>    def register_blueprints(app):\n        app.register_blueprint(new_blueprint)\n</code></pre>"},{"location":"contributing/website/Flask/#database-setup","title":"Database setup","text":"<p>If you want o setup a local user database similar to OpenML then follow these steps:</p> <ol> <li>Install MySQL</li> <li>Create a new database 'openml'</li> <li>Set current database to 'openml' via use method</li> <li>Download users.sql file from openml.org github repo and add it in the openml db via \"mysql -u root -p openml &lt; users.sql\"</li> <li>Edit the database path in `server/extensions.py` and `server/config.py`</li> </ol> <p>Note: Remember to add passwords and socket extension address(if any) in both in <code>server/extensions.py</code> and <code>server/config.py</code> </p>"},{"location":"contributing/website/Flask/#security","title":"Security","text":"<p>Flask backend uses JSON web tokens for all the user handling tasks. Flask JWT extended library is used to bind JWT with the flask app. Current Mechanism is :</p> <ol> <li> User logs in.</li> <li> JWT token is assigned to user and sent with every request to frontend.</li> <li> All the user information can only be accessed with a JWT token like edit profile and API-key.</li> <li> The JWT token is stored in local memory of the browser.</li> <li> The token get expired after 2 hours or get blacklisted after logout.</li> </ol> <p>JWT is registered as an extension in `server/extensions.py`. All the user password hash are saved in Argon2 format with the new backend.</p>"},{"location":"contributing/website/Flask/#registering-extensions","title":"Registering Extensions","text":"<p>To register a new extension to flask backend extension has to be added in <code>server/extensions.py</code> and initialized in server/app.py. Current extensions are : flask_argon2, flask_bcrypt, flask_jwt_extended and flask_sqlalchemy.</p>"},{"location":"contributing/website/Flask/#configuring-app","title":"Configuring App","text":"<p>Configuration variables like secret keys, Database URI and extension configurations are specified in  <code>server/config.py</code> with Config object, which is supplied to the flask app during initialization.</p>"},{"location":"contributing/website/Flask/#creating-a-new-route","title":"Creating a new route","text":"<p>To create a new route in backend you can add the route in <code>server/public/views.py</code> or <code>server/user/views.py</code> (if it requires user authorisation or JWT usage in any way).  </p>"},{"location":"contributing/website/Flask/#bindings-to-openml-server","title":"Bindings to OpenML server","text":"<p>You can specify which OpenML server to connect to. This is stored in the <code>.env</code> file in the main directory. It is set to the main OpenML server by default:</p> <pre><code>    ELASTICSEARCH_SERVER=https://www.openml.org/es\n    OPENML_SERVER=https://www.openml.org\n</code></pre> <p>The ElasticSearch server is used to download information about datasets, tasks, flows and runs, as well as to power the frontend search. The OpenML server is used for uploading datasets, tasks, and anything else that requires calls to the OpenML API.</p>"},{"location":"contributing/website/Flask/#bindings-to-frontend","title":"Bindings to frontend","text":"<p>The frontend is generated by React. See below for more information. The React app is loaded as a static website. This is done in Flask setup in file <code>server.py</code>.</p> <pre><code>    app = Flask(__name__, static_url_path='', static_folder='src/client/app/build')\n</code></pre> <p>It will find the React app there and load it.</p>"},{"location":"contributing/website/Flask/#email-server","title":"Email Server","text":"<p>OpenML uses its own mail server, You can use basically any mail server compatible with python SMTP library. Our suggestion is to use mailtrap.io for local testing. You can configure email server configurations in .env file. Currently we only use emails for confirmation email and forgotten password emails.</p>"},{"location":"contributing/website/React/","title":"React App","text":""},{"location":"contributing/website/React/#app-structure","title":"App structure","text":"<p>The structure of the source code looks as follows</p> <pre><code>App.js\nindex.js\ncomponents\n|-- Sidebar.js\n|-- Header.js\n|-- ...\nlayouts\n|-- Clear.js\n|-- Main.js\npages\n|-- auth\n|-- cover\n|-- docs\n|-- search\nroutes\n|-- index.js\n|-- Routes.js\nthemes\n</code></pre> <p>The website is designed as a single-page application. The top level files bootstrap the app. <code>index.js</code> simply renders the top component, and <code>App.js</code> adds the relevant subcomponents based on the current theme and state.</p> <p><code>Routes.js</code> links components to the possible routes (based on the URL). The list of possible routes is defined in <code>routes/index.js</code>.</p> <p><code>pages</code> contain the various pages of the website. It has subdirectories for:</p> <ul> <li><code>auth</code>: All pages that require authorization (login). These routes are protected.</li> <li><code>cover</code>: The front page of the website</li> <li><code>docs</code>: All normal information pages (e.g. 'About', 'API',...)</li> <li><code>search</code>: All pages related to searching for datasets, tasks, flows, runs, etc.</li> </ul> <p><code>layout</code> contains the possible layouts, <code>Main</code> or <code>Clear</code> (see below). You define the layout of a page by adding its route to either <code>mainRoutes</code> or <code>clearRoutes</code> in <code>routes/index.js</code>. The default is the <code>Main</code> layout.</p> <p><code>themes</code> contains the overall theme styling for the entire website. Currently, there is a dark and a light theme. They can be set using <code>setTheme</code> in the MainContext, see <code>App.js</code>.</p>"},{"location":"contributing/website/React/#component-structure","title":"Component structure","text":"<p>The component structure is shown above, for the <code>Main</code> layout. The <code>App</code> component also holds the state of the website using React's native Context API (see below). Next to the header and sidebar, the main component of the website (in yellow) shows the contents of the current <code>page</code>. In this image, this is the search page, which has several subcomponents as explained below.</p>"},{"location":"contributing/website/React/#search-page","title":"Search page","text":"<p>The search page is structured as follows:</p> <ul> <li> <p><code>SearchPanel</code>: the main search panel. Also contains callbacks for sorting and filtering, and lists what can be filtered or sorted on.</p> </li> <li> <p><code>FilterBar</code>: The top bar with the search statistics and functionality to add filters and sort results</p> </li> <li> <p><code>SearchResultsPanel</code>: The list of search results on the left. It shows a list of <code>Card</code> elements which are uniformly styled but their contents may vary. Depending on the selected type of result (selected in the left navigation bar) it is instantiated with different properties. E.g. a <code>DataListPanel</code> is a simple wrapper around <code>SearchResultsPanel</code> which defines the dataset-specific statistics to be shown in the cards.</p> <ul> <li>Search tabs: The tabs that allow you to choose between different aspects of the results (Statistics, Overview (Dash)) or the different views on the selected dataset, task, etc. (Details, Analysis (Dash),...)</li> <li><code>ItemDetail</code>: When a search result is selected, this will show the details of the selection, e.g. the dataset details. Depending on the passed <code>type</code> prop, it will render the <code>Dataset</code>, <code>Task</code>, ... component.</li> </ul> </li> </ul> <p>The <code>api.js</code> file contains the <code>search</code> function, which translates a search query, filters, and other constraints into an ElasticSearch query and returns the results.</p>"},{"location":"contributing/website/React/#style-guide","title":"Style guide","text":"<p>To keep a consistent style and minimize dependencies and complexity, we build on Material UI components and FontAwesome icons. Theming is defined in <code>themes/index.js</code> and loaded in as a context (<code>ThemeContext</code>) in <code>App.js</code>. More specific styling is always defined through styled components in the corresponding pages.</p>"},{"location":"contributing/website/React/#layouts","title":"Layouts","text":"<p>There are two top level layouts: <code>Main</code> loads the main layout with a <code>Sidebar</code>, <code>Header</code>, and a certain page with all the contents. The <code>Clear.js</code> layout has no headers or sidebars, but has a colored gradient background. It is used mainly for user login and registration or other quick forms.</p> <p>The layout of the page content should use the Material UI grid layout. This makes sure it will adapt to different device screen sizes. Test using your browsers development tools whether the layout adapts correctly to different screens, including recent smartphones.</p>"},{"location":"contributing/website/React/#styled-components","title":"Styled components","text":"<p>Any custom styling (beyond the Material UI default styling) is defined in styled components which are defined within the file for each page. Keep this as minimal as possible. Check if you can import styled components already defined for other pages, avoid duplication.</p> <p>Styled div's are defined as follows:</p> <pre><code>const OpenMLTitle = styled.div`\n  color: white;\n  font-size: 3em;\n`;\n</code></pre> <p>Material UI components can be styled the same way:</p> <pre><code>const WhiteButton = styled(Button)`\n  display: inline-block;\n  color: #fff;\n`;\n</code></pre>"},{"location":"contributing/website/React/#color-palette","title":"Color palette","text":"<p>We follow the general Material UI color palette with shade 400, except when that doesn't give sufficient contrast. The main colors used (e.g. for the icons in the sidebar are: 'green[400]', 'yellow[700]', 'blue[800]', 'red[400]', 'purple[400]', 'orange[400]', 'grey[400]'. Backgrounds are generally kept white (or dark grey for the dark theme). The global context (see below) has a <code>getColor</code> function to get the colors of the search types, e.g. <code>context.getColor(\"run\")</code> returns <code>red[400]</code>.</p>"},{"location":"contributing/website/React/#handling-state","title":"Handling state","text":"<p>There are different levels of state management:</p> <ul> <li>Global state is handled via React's native Context API (we don't use Redux). Contexts are defined in the component tree where needed (usually higher up) by a context provider component, and is accessed lower in the component tree by a context consumer. For instance, see the <code>ThemeContext.Provider</code> in <code>App.js</code> and the <code>ThemeContext.Consumer</code> in <code>Sidebar.js</code>. There is a <code>MainContext</code> which contains global state values such as the logged in user details, and the current state of the search.</li> <li>Lower level components can pass state to their child components via props.</li> <li>Local state changes should, when possible, be defined by React Hooks.</li> </ul> <p>Note that changing the global state will re-render the entire website. Hence, do this only when necessary.</p>"},{"location":"contributing/website/React/#state-and-search","title":"State and search","text":"<p>Most global state variables have to do with search. The search pages typically work by changing the <code>query</code> and <code>filters</code> variables (see <code>App.js</code>). There is a <code>setSearch</code> function in the main context that can be called to change the search parameters. It checks whether the query has changed and whether updating the global state and re-rendering the website is necessary.</p>"},{"location":"contributing/website/React/#lifecycle-methods","title":"Lifecycle Methods","text":"<p>These are the React lifecycle methods and how we use them. When a component mounts, methods 1,2,4,7 will be called. When it updates, methods 2-6 will be called.</p> <ol> <li>constructor(): Set the initial state of the components</li> <li>getDerivedStateFromProps(props, state): Static method, only for changing the local state based on props. It returns the new state.</li> <li>shouldComponentUpdate(nextProps, nextState): Decides whether a state change requires a re-rendering or not. Used to optimize performance.</li> <li>render(): Returns the JSX to be rendered. It should NOT change the state.</li> <li>getSnapshotBeforeUpdate(prevProps,prevState): Used to save 'old' DOM information right before an update. Returns a 'snapshot'.</li> <li>componentDidUpdate(prevProps,prevState,snapshot): For async requests or other operations right after component update.</li> <li>componentDidMount(): For async requests (e.g. API calls) right after the component mounted.</li> <li>componentWillUnMount(): Cleanup before the component is destroyed.</li> <li>componentDidCatch(error,info): For updating the state after an error is thrown.</li> </ol>"},{"location":"contributing/website/React/#forms-and-events","title":"Forms and Events","text":"<p>React wraps native browser events into synthetic events to handle interactions in a cross-browser compatible way. After being wrapped, they are sent to all event handlers, usually defined as callbacks. Note: for performance reasons, synthetic events are pooled and reused, so their properties are nullified after being consumed. If you want to use them asynchronously, you need to call <code>event.persist()</code>.</p> <p>HTML forms are different than other DOM elements because they keep their own state in plain HTML. To make sure that we can control the state we need to set the input field's <code>value</code> to a component state value.</p> <p>Here's an example of using an input field to change the title displayed in the component.</p> <pre><code>const titles: {mainTitle: 'OpenML'};\n\nclass App extends Component {\n  this.state = {titles};\n\n  // Receive synthetic event\n  onTitleChange = (event) =&gt; {\n    this.setState({titles.mainTitle : event.target.value});\n  }\n\n  render(){\n    return (\n      &lt;div classname=\"App\"&gt;\n        &lt;h1&gt;{this.state.titles.mainTitle}&lt;/h1&gt;\n        &lt;form&gt;\n          &lt;input type=\"text\"\n          value={this.state.titles.mainTitle} // control state\n          onChange={this.onTitleChange} // event handler callback\n          /&gt;\n        &lt;/form&gt;\n      &lt;/div&gt;\n    );\n  }\n}\n</code></pre>"},{"location":"contributing/website/Website/","title":"Getting started","text":""},{"location":"contributing/website/Website/#installation","title":"Installation","text":"<p>The OpenML website runs on Flask, React, and Dash. You need to install these first.</p> <ul> <li> <p>Download or clone the source code for the OpenML website from GitHub. Then, go into that folder (it should have the <code>requirements.txt</code> and <code>package.json</code> files). <pre><code>git clone https://github.com/openml/openml.org.git\ncd openml.org\n</code></pre></p> </li> <li> <p>Install Flask, Dash, and dependencies using PIP <pre><code>pip install -r requirements.txt\n</code></pre></p> </li> <li> <p>Install React and dependencies using NPM (8 or higher) <pre><code>cd server/src/client/app/\nnpm install\n</code></pre></p> </li> </ul>"},{"location":"contributing/website/Website/#building-and-running","title":"Building and running","text":"<p>Go back to the home directory. Build a production version of the website with:</p> <pre><code>npm run build --prefix server/src/client/app/\n</code></pre> <p>Start the server by running:</p> <pre><code>flask run\n</code></pre> <p>You should now see the app running in your browser at <code>localhost:5000</code></p> <p>Note: If you run the app using HTTPS, add the SSL context or use 'adhoc' to use on-the-fly certificates or you can specify your own certificates.</p> <pre><code>flask run --cert='adhoc'\n</code></pre> <p>As flask server is not suitable for production we recommend you to use some other server if you want to deploy your openml installation in production. We currently use gunicorn for production server. You can install the gunicorn server and run it: <pre><code>gunicorn --certfile cert.pem --keyfile key.pem -b localhost:5000 autoapp:app\n</code></pre></p>"},{"location":"contributing/website/Website/#development","title":"Development","text":"<p>To start the React frontend in developer mode, go to <code>server/src/client/app</code> and run:</p> <pre><code>npm run start\n</code></pre> <p>The app should automatically open at <code>localhost:3000</code> and any changes made to the code will automatically reload the website (hot loading).</p> <p>For the new Next.js frontend, install and run like this: <pre><code>cd app\nnpm install\nnpm run dev\n</code></pre></p>"},{"location":"contributing/website/Website/#structure","title":"Structure","text":"<p>The website is built on the following components:  </p> <ul> <li>A Flask backend. Written in Python, the backend takes care of all communication with the OpenML server. It builds on top of the OpenML Python API. It also takes care of user authentication and keeps the search engine (ElasticSearch) up to date with the latest information from the server. Files are located in the <code>server</code> folder.</li> <li>A React frontend. Written in JavaScript, this takes care of rendering the website. It pulls in information from the search engine, and shows plots rendered by Dash. It also contains forms (e.g. for logging in or uploading new datasets), which will be sent off to the backend for processing. Files are located in <code>server/src/client/app</code>.</li> <li>Dash dashboards. Written in Python, Dash is used for writing interactive plots. It pulls in data from the Python API, and renders the plots as React components. Files are located in <code>server/src/dashboard</code>.</li> </ul>"},{"location":"ecosystem/","title":"Ecosystem","text":"<p>OpenML has a rich ecosystem of tools and projects that seamlessly integrate OpenML in various ways. </p> <p>Add your library</p> <p>Did you use OpenML in your work and want to share it with the community? We would love to have you! Simply create a pull request with the necessary information (click the  icon) and we will add it to this page.</p> <p>Integrate OpenML in your libraries</p> <p>If you want to integrate OpenML into machine learning and data science tools, it's easiest to build on one of the existing clients,  which often can be used as is or extended. For instance, see how to extend the Python API to integrate OpenML into Python tools. </p> automlbenchmark  402 stars <p>OpenML AutoML Benchmarking Framework</p> openml-python  280 stars <p>OpenML's Python API for a World of Data and More \ud83d\udcab</p> openml-r  95 stars <p>R package to interface with OpenML</p> openml-rust  11 stars <p>A rust interface to http://openml.org/</p> OpenML.jl  10 stars <p>Partial implementation of the OpenML API for Julia</p> continual-automl  5 stars <p>Adaptations of AutoML libraries H2O, Autosklearn and GAMA for stream learning</p> openml-pytorch  5 stars <p>Pytorch extension for openml-python</p> openml-dotnet  5 stars <p>.NET API</p> openml-rapidminer  4 stars <p>RapidMiner plugin</p> openml-tensorflow  2 stars <p>Tensorflow extension for openml-python</p> OpenmlCortana  0 stars <p>Openml Cortana connector</p> openml-keras  0 stars <p>Keras extension for openml-python</p> openml-croissant  0 stars <p>Converting dataset metadata from OpenML to Croissant format</p> openml-azure  0 stars <p>Tools for interfacing with Azure</p> openml-onnx  0 stars <p>onnx extension for openml</p> openml-mxnet  0 stars <p>MXNet extension for openml</p> flow-visualization  0 stars <p>Tool to convert openml flows to ONNX and visualize them via Netron</p>"},{"location":"ecosystem/Java/","title":"Java","text":"<p>The Java API allows you connect to OpenML from Java applications.</p>"},{"location":"ecosystem/Java/#java-docs","title":"Java Docs","text":"<p>Read the full Java Docs.</p>"},{"location":"ecosystem/Java/#download","title":"Download","text":"<p>Stable releases of the Java API are available from Maven Central Or, you can check out the developer version from GitHub</p> <p>Include the jar file in your projects as usual, or install via Maven.</p>"},{"location":"ecosystem/Java/#quick-start","title":"Quick Start","text":"<ul> <li>Create an <code>OpenmlConnector</code> instance with your authentication details. This will create a client with all OpenML functionalities. <p>OpenmlConnector client = new OpenmlConnector(\"api_key\")</p> </li> </ul> <p>All functions are described in the Java Docs.</p>"},{"location":"ecosystem/Java/#downloading","title":"Downloading","text":"<p>To download data, flows, tasks, runs, etc. you need the unique id of that resource. The id is shown on each item's webpage and in the corresponding url. For instance, let's download Data set 1. The following returns a DataSetDescription object that contains all information about that data set.</p> <pre><code>DataSetDescription data = client.dataGet(1);\n</code></pre> <p>You can also search for the items you need online, and click the icon to get all id's that match a search.</p>"},{"location":"ecosystem/Java/#uploading","title":"Uploading","text":"<p>To upload data, flows, runs, etc. you need to provide a description of the object. We provide wrapper classes to provide this information, e.g. <code>DataSetDescription</code>, as well as to capture the server response, e.g. <code>UploadDataSet</code>, which always includes the generated id for reference:</p> <pre><code>DataSetDescription description = new DataSetDescription( \"iris\", \"The famous iris dataset\", \"arff\", \"class\");\nUploadDataSet result = client.dataUpload( description, datasetFile );\nint data_id = result.getId();\n</code></pre> <p>More details are given in the corresponding functions below. Also see the Java Docs for all possible inputs and return values.</p>"},{"location":"ecosystem/Java/#data-download","title":"Data download","text":""},{"location":"ecosystem/Java/#datagetint-data_id","title":"<code>dataGet(int data_id)</code>","text":"<p>Retrieves the description of a specified data set.</p> <pre><code>DataSetDescription data = client.dataGet(1);\nString name = data.getName();\nString version = data.getVersion();\nString description = data.getDescription();\nString url = data.getUrl();\n</code></pre>"},{"location":"ecosystem/Java/#datafeaturesint-data_id","title":"<code>dataFeatures(int data_id)</code>","text":"<p>Retrieves the description of the features of a specified data set.</p> <pre><code>DataFeature reponse = client.dataFeatures(1);\nDataFeature.Feature[] features = reponse.getFeatures();\nString name = features[0].getName();\nString type = features[0].getDataType();\nboolean isTarget = features[0].getIs_target();\n</code></pre>"},{"location":"ecosystem/Java/#dataqualityint-data_id","title":"<code>dataQuality(int data_id)</code>","text":"<p>Retrieves the description of the qualities (meta-features) of a specified data set.</p> <pre><code>    DataQuality response = client.dataQuality(1);\n    DataQuality.Quality[] qualities = reponse.getQualities();\n    String name = qualities[0].getName();\n    String value = qualities[0].getValue();\n</code></pre>"},{"location":"ecosystem/Java/#dataqualityint-data_id-int-start-int-end-int-interval_size","title":"<code>dataQuality(int data_id, int start, int end, int interval_size)</code>","text":"<p>For data streams. Retrieves the description of the qualities (meta-features) of a specified portion of a data stream.</p> <pre><code>    DataQuality qualities = client.dataQuality(1,0,10000,null);\n</code></pre>"},{"location":"ecosystem/Java/#dataqualitylist","title":"<code>dataQualityList()</code>","text":"<p>Retrieves a list of all data qualities known to OpenML.</p> <pre><code>    DataQualityList response = client.dataQualityList();\n    String[] qualities = response.getQualities();\n</code></pre>"},{"location":"ecosystem/Java/#data-upload","title":"Data upload","text":""},{"location":"ecosystem/Java/#datauploaddatasetdescription-description-file-dataset","title":"<code>dataUpload(DataSetDescription description, File dataset)</code>","text":"<p>Uploads a data set file to OpenML given a description. Throws an exception if the upload failed, see openml.data.upload for error codes.</p> <pre><code>    DataSetDescription dataset = new DataSetDescription( \"iris\", \"The iris dataset\", \"arff\", \"class\");\n    UploadDataSet data = client.dataUpload( dataset, new File(\"data/path\"));\n    int data_id = result.getId();\n</code></pre>"},{"location":"ecosystem/Java/#datauploaddatasetdescription-description","title":"<code>dataUpload(DataSetDescription description)</code>","text":"<p>Registers an existing dataset (hosted elsewhere). The description needs to include the url of the data set. Throws an exception if the upload failed, see openml.data.upload for error codes.</p> <pre><code>    DataSetDescription description = new DataSetDescription( \"iris\", \"The iris dataset\", \"arff\", \"class\");\n    description.setUrl(\"http://datarepository.org/mydataset\");\n    UploadDataSet data = client.dataUpload( description );\n    int data_id = result.getId();\n</code></pre>"},{"location":"ecosystem/Java/#flow-download","title":"Flow download","text":""},{"location":"ecosystem/Java/#flowgetint-flow_id","title":"<code>flowGet(int flow_id)</code>","text":"<p>Retrieves the description of the flow/implementation with the given id.</p> <pre><code>    Implementation flow = client.flowGet(100);\n    String name = flow.getName();\n    String version = flow.getVersion();\n    String description = flow.getDescription();\n    String binary_url = flow.getBinary_url();\n    String source_url = flow.getSource_url();\n    Parameter[] parameters = flow.getParameter();\n</code></pre>"},{"location":"ecosystem/Java/#flow-management","title":"Flow management","text":""},{"location":"ecosystem/Java/#flowowned","title":"<code>flowOwned()</code>","text":"<p>Retrieves an array of id's of all flows/implementations owned by you.</p> <pre><code>    ImplementationOwned response = client.flowOwned();\n    Integer[] ids = response.getIds();\n</code></pre>"},{"location":"ecosystem/Java/#flowexistsstring-name-string-version","title":"<code>flowExists(String name, String version)</code>","text":"<p>Checks whether an implementation with the given name and version is already registered on OpenML.</p> <pre><code>    ImplementationExists check = client.flowExists(\"weka.j48\", \"3.7.12\");\n    boolean exists = check.exists();\n    int flow_id = check.getId();\n</code></pre>"},{"location":"ecosystem/Java/#flowdeleteint-id","title":"<code>flowDelete(int id)</code>","text":"<p>Removes the flow with the given id (if you are its owner).</p> <pre><code>    ImplementationDelete response = client.openmlImplementationDelete(100);\n</code></pre>"},{"location":"ecosystem/Java/#flow-upload","title":"Flow upload","text":""},{"location":"ecosystem/Java/#flowuploadimplementation-description-file-binary-file-source","title":"<code>flowUpload(Implementation description, File binary, File source)</code>","text":"<p>Uploads implementation files (binary and/or source) to OpenML given a description.</p> <pre><code>    Implementation flow = new Implementation(\"weka.J48\", \"3.7.12\", \"description\", \"Java\", \"WEKA 3.7.12\")\n    UploadImplementation response = client.flowUpload( flow, new File(\"code.jar\"), new File(\"source.zip\"));\n    int flow_id = response.getId();\n</code></pre>"},{"location":"ecosystem/Java/#task-download","title":"Task download","text":""},{"location":"ecosystem/Java/#taskgetint-task_id","title":"<code>taskGet(int task_id)</code>","text":"<p>Retrieves the description of the task with the given id.</p> <pre><code>    Task task = client.taskGet(1);\n    String task_type = task.getTask_type();\n    Input[] inputs = task.getInputs();\n    Output[] outputs = task.getOutputs();\n</code></pre>"},{"location":"ecosystem/Java/#taskevaluationsint-task_id","title":"<code>taskEvaluations(int task_id)</code>","text":"<p>Retrieves all evaluations for the task with the given id.</p> <pre><code>    TaskEvaluations response = client.taskEvaluations(1);\n    Evaluation[] evaluations = response.getEvaluation();\n</code></pre>"},{"location":"ecosystem/Java/#taskevaluationsint-task_id-int-start-int-end-int-interval_size","title":"<code>taskEvaluations(int task_id, int start, int end, int interval_size)</code>","text":"<p>For data streams. Retrieves all evaluations for the task over the specified window of the stream.</p> <pre><code>    TaskEvaluations response = client.taskEvaluations(1);\n    Evaluation[] evaluations = response.getEvaluation();\n</code></pre>"},{"location":"ecosystem/Java/#run-download","title":"Run download","text":""},{"location":"ecosystem/Java/#rungetint-run_id","title":"<code>runGet(int run_id)</code>","text":"<p>Retrieves the description of the run with the given id.</p> <pre><code>    Run run = client.runGet(1);\n    int task_id = run.getTask_id();\n    int flow_id = run.getImplementation_id();\n    Parameter_setting[] settings = run.getParameter_settings()\n    EvaluationScore[] scores = run.getOutputEvaluation();\n</code></pre>"},{"location":"ecosystem/Java/#run-management","title":"Run management","text":""},{"location":"ecosystem/Java/#rundeleteint-run_id","title":"<code>runDelete(int run_id)</code>","text":"<p>Deletes the run with the given id (if you are its owner).</p> <pre><code>    RunDelete response = client.runDelete(1);\n</code></pre>"},{"location":"ecosystem/Java/#run-upload","title":"Run upload","text":""},{"location":"ecosystem/Java/#runuploadrun-description-mapstringfile-output_files","title":"<code>runUpload(Run description, Map&lt;String,File&gt; output_files)</code>","text":"<p>Uploads a run to OpenML, including a description and a set of output files depending on the task type.</p> <pre><code>    Run.Parameter_setting[] parameter_settings = new Run.Parameter_setting[1];\n    parameter_settings[0] = Run.Parameter_setting(null, \"M\", \"2\");\n    Run run = new Run(\"1\", null, \"100\", \"setup_string\", parameter_settings);\n    Map outputs = new HashMap&lt;String,File&gt;();\n    outputs.add(\"predictions\",new File(\"predictions.arff\"));\n    UploadRun response = client.runUpload( run, outputs);\n    int run_id = response.getRun_id();\n</code></pre>"},{"location":"ecosystem/MOA/","title":"MOA","text":"<p>OpenML features extensive support for MOA. However currently this is implemented as a stand alone MOA compilation, using the latest version (as of May, 2014).</p> <p>Download MOA for OpenML</p>"},{"location":"ecosystem/MOA/#quick-start","title":"Quick Start","text":"<ul> <li>Download the standalone MOA environment above.</li> <li>Find your API key in your profile (log in first). Create a config file called <code>openml.conf</code> in a <code>.openml</code> directory in your home dir. It should contain the following lines: <p>api_key = YOUR_KEY</p> </li> <li>Launch the JAR file by double clicking on it, or launch from command-line using the following command: <p>java -cp openmlmoa.beta.jar moa.gui.GUI</p> </li> <li>Select the task <code>moa.tasks.openml.OpenmlDataStreamClassification</code> to evaluate a classifier on an OpenML task, and send the results to OpenML.</li> <li>Optionally, you can generate new streams using the Bayesian Network Generator: select the <code>moa.tasks.WriteStreamToArff</code> task, with <code>moa.streams.generators.BayesianNetworkGenerator</code>.</li> </ul>"},{"location":"ecosystem/Python_extensions/","title":"Integrating your Python libraries","text":"<p>OpenML-Python provides an extension interface to connect other machine learning libraries than scikit-learn to OpenML. Please check the <code>api_extensions</code> and use the scikit-learn extension in <code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text role=\"class\"} as a starting point.</p>"},{"location":"ecosystem/Python_extensions/#connecting-new-machine-learning-libraries","title":"Connecting new machine learning libraries","text":""},{"location":"ecosystem/Python_extensions/#content-of-the-library","title":"Content of the Library","text":"<p>To leverage support from the community and to tap in the potential of OpenML, interfacing with popular machine learning libraries is essential. The OpenML-Python package is capable of downloading meta-data and results (data, flows, runs), regardless of the library that was used to upload it. However, in order to simplify the process of uploading flows and runs from a specific library, an additional interface can be built. The OpenML-Python team does not have the capacity to develop and maintain such interfaces on its own. For this reason, we have built an extension interface to allows others to contribute back. Building a suitable extension for therefore requires an understanding of the current OpenML-Python support.</p> <p>The <code>sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py</code>{.interpreted-text role=\"ref\"} tutorial shows how scikit-learn currently works with OpenML-Python as an extension. The sklearn extension packaged with the openml-python repository can be used as a template/benchmark to build the new extension.</p>"},{"location":"ecosystem/Python_extensions/#api","title":"API","text":"<ul> <li>The extension scripts must import the [openml]{.title-ref} package     and be able to interface with any function from the OpenML-Python     <code>api</code>.</li> <li>The extension has to be defined as a Python class and must inherit     from <code>openml.extensions.Extension</code>.</li> <li>This class needs to have all the functions from [class     Extension]{.title-ref} overloaded as required.</li> <li>The redefined functions should have adequate and appropriate     docstrings. The [Sklearn Extension API     :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref}     is a good example to follow.</li> </ul>"},{"location":"ecosystem/Python_extensions/#interfacing-with-openml-python","title":"Interfacing with OpenML-Python","text":"<p>Once the new extension class has been defined, the openml-python module to <code>openml.extensions.register_extension</code> must be called to allow OpenML-Python to interface the new extension.</p> <p>The following methods should get implemented. Although the documentation in the [Extension]{.title-ref} interface should always be leading, here we list some additional information and best practices. The [Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is a good example to follow. Note that most methods are relatively simple and can be implemented in several lines of code.</p> <ul> <li>General setup (required)<ul> <li><code>can_handle_flow</code>: Takes as     argument an OpenML flow, and checks whether this can be handled     by the current extension. The OpenML database consists of many     flows, from various workbenches (e.g., scikit-learn, Weka, mlr).     This method is called before a model is being deserialized.     Typically, the flow-dependency field is used to check whether     the specific library is present, and no unknown libraries are     present there.</li> <li><code>can_handle_model</code>: Similar as     <code>can_handle_flow</code>, except that in     this case a Python object is given. As such, in many cases, this     method can be implemented by checking whether this adheres to a     certain base class.</li> </ul> </li> <li>Serialization and De-serialization (required)<ul> <li><code>flow_to_model</code>: deserializes the     OpenML Flow into a model (if the library can indeed handle the     flow). This method has an important interplay with     <code>model_to_flow</code>. Running these     two methods in succession should result in exactly the same     model (or flow). This property can be used for unit testing     (e.g., build a model with hyperparameters, make predictions on a     task, serialize it to a flow, deserialize it back, make it     predict on the same task, and check whether the predictions are     exactly the same.) The example in the scikit-learn interface     might seem daunting, but note that here some complicated design     choices were made, that allow for all sorts of interesting     research questions. It is probably good practice to start easy.</li> <li><code>model_to_flow</code>: The inverse of     <code>flow_to_model</code>. Serializes a     model into an OpenML Flow. The flow should preserve the class,     the library version, and the tunable hyperparameters.</li> <li><code>get_version_information</code>: Return     a tuple with the version information of the important libraries.</li> <li><code>create_setup_string</code>: No longer     used, and will be deprecated soon.</li> </ul> </li> <li>Performing runs (required)<ul> <li><code>is_estimator</code>: Gets as input a     class, and checks whether it has the status of estimator in the     library (typically, whether it has a train method and a predict     method).</li> <li><code>seed_model</code>: Sets a random seed     to the model.</li> <li><code>_run_model_on_fold</code>: One of the     main requirements for a library to generate run objects for the     OpenML server. Obtains a train split (with labels) and a test     split (without labels) and the goal is to train a model on the     train split and return the predictions on the test split. On top     of the actual predictions, also the class probabilities should     be determined. For classifiers that do not return class     probabilities, this can just be the hot-encoded predicted label.     The predictions will be evaluated on the OpenML server. Also,     additional information can be returned, for example,     user-defined measures (such as runtime information, as this can     not be inferred on the server). Additionally, information about     a hyperparameter optimization trace can be provided.</li> <li><code>obtain_parameter_values</code>:     Obtains the hyperparameters of a given model and the current     values. Please note that in the case of a hyperparameter     optimization procedure (e.g., random search), you only should     return the hyperparameters of this procedure (e.g., the     hyperparameter grid, budget, etc) and that the chosen model will     be inferred from the optimization trace.</li> <li><code>check_if_model_fitted</code>: Check     whether the train method of the model has been called (and as     such, whether the predict method can be used).</li> </ul> </li> <li>Hyperparameter optimization (optional)<ul> <li><code>instantiate_model_from_hpo_class</code>{.interpreted-text     role=\"meth\"}: If a given run has recorded the hyperparameter     optimization trace, then this method can be used to     reinstantiate the model with hyperparameters of a given     hyperparameter optimization iteration. Has some similarities     with <code>flow_to_model</code> (as this     method also sets the hyperparameters of a model). Note that     although this method is required, it is not necessary to     implement any logic if hyperparameter optimization is not     implemented. Simply raise a [NotImplementedError]{.title-ref}     then.</li> </ul> </li> </ul>"},{"location":"ecosystem/Python_extensions/#hosting-the-library","title":"Hosting the library","text":"<p>Each extension created should be a stand-alone repository, compatible with the OpenML-Python repository. The extension repository should work off-the-shelf with OpenML-Python installed.</p> <p>Create a public Github repo with the following directory structure:</p> <pre><code>| [repo name]\n|    |-- [extension name]\n|    |    |-- __init__.py\n|    |    |-- extension.py\n|    |    |-- config.py (optionally)\n</code></pre>"},{"location":"ecosystem/Python_extensions/#recommended","title":"Recommended","text":"<ul> <li>Test cases to keep the extension up to date with the     [openml-python]{.title-ref} upstream changes.</li> <li>Documentation of the extension API, especially if any new     functionality added to OpenML-Python\\'s extension design.</li> <li>Examples to show how the new extension interfaces and works with     OpenML-Python.</li> <li>Create a PR to add the new extension to the OpenML-Python API     documentation.</li> </ul> <p>Happy contributing!</p>"},{"location":"ecosystem/Rest/","title":"REST tutorial","text":"<p>OpenML offers a RESTful Web API, with predictive URLs, for uploading and downloading machine learning resources. Try the API Documentation to see examples of all calls, and test them right in your browser.</p>"},{"location":"ecosystem/Rest/#getting-started","title":"Getting started","text":"<ul> <li>REST services can be called using simple HTTP GET or POST actions.</li> <li>The REST Endpoint URL is <code>https://www.openml.org/api/v1/</code></li> <li>The default endpoint returns data in XML. If you prefer JSON, use the endpoint <code>https://www.openml.org/api/v1/json/</code>. Note that, to upload content, you still need to use XML (at least for now).</li> </ul>"},{"location":"ecosystem/Rest/#testing","title":"Testing","text":"<p>For continuous integration and testing purposes, we have a test server offering the same API, but which does not affect the production server.</p> <ul> <li>The test server REST Endpoint URL is <code>https://test.openml.org/api/v1/</code></li> </ul>"},{"location":"ecosystem/Rest/#error-messages","title":"Error messages","text":"<p>Error messages will look like this:</p> <pre><code>&lt;oml:error xmlns:oml=\"http://openml.org/error\"&gt;\n&lt;oml:code&gt;100&lt;/oml:code&gt;\n&lt;oml:message&gt;Please invoke legal function&lt;/oml:message&gt;\n&lt;oml:additional_information&gt;Additional information, not always available.&lt;/oml:additional_information&gt;\n&lt;/oml:error&gt;\n</code></pre> <p>All error messages are listed in the API documentation. E.g. try to get a non-existing dataset:</p> <ul> <li>in XML: https://www.openml.org/api_new/v1/data/99999</li> <li>in JSON: https://www.openml.org/api_new/v1/json/data/99999</li> </ul>"},{"location":"ecosystem/Rest/#examples","title":"Examples","text":"<p>You need to be logged in for these examples to work.</p>"},{"location":"ecosystem/Rest/#download-a-dataset","title":"Download a dataset","text":"<ul> <li>User asks for a dataset using the /data/{id} service. The <code>dataset id</code> is typically part of a task, or can be found on OpenML.org.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting the dataset. This can be OpenML, but also any other data repository. Try it now</li> </ul>"},{"location":"ecosystem/Rest/#download-a-flow","title":"Download a flow","text":"<ul> <li>User asks for a flow using the /flow/{id} service and a <code>flow id</code>. The <code>flow id</code> can be found on OpenML.org.</li> <li>OpenML returns a description of the flow as an XML file (or JSON). Try it now</li> <li>The flow description contains the URL where the flow can be downloaded (e.g. GitHub), either as source, binary or both, as well as additional information on history, dependencies and licence. The user calls the right URL to download it.</li> <li>The flow is returned by the server hosting it. This can be OpenML, but also any other code repository. Try it now</li> </ul>"},{"location":"ecosystem/Rest/#download-a-task","title":"Download a task","text":"<ul> <li>User asks for a task using the /task/{id} service and a <code>task id</code>. The <code>task id</code> is typically returned when searching for tasks.</li> <li>OpenML returns a description of the task as an XML file (or JSON). Try it now</li> <li>The task description contains the <code>dataset id</code>(s) of the datasets involved in this task. The user asks for the dataset using the /data/{id} service and the <code>dataset id</code>.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting it. This can be OpenML, but also any other data repository. Try it now</li> <li>The task description may also contain links to other resources, such as the train-test splits to be used in cross-validation. The user calls that URL to download the train-test splits.</li> <li>The train-test splits are returned by OpenML. Try it now</li> </ul>"},{"location":"ecosystem/Weka/","title":"Weka","text":"<p>OpenML is integrated in the Weka (Waikato Environment for Knowledge Analysis) Experimenter and the Command Line Interface.</p>"},{"location":"ecosystem/Weka/#installation","title":"Installation","text":"<p>OpenML is available as a weka extension in the package manager:</p> <ul> <li>Download the latest version (3.7.13 or higher).</li> <li>Launch Weka, or start from commandline: <p>java -jar weka.jar</p> </li> <li>If you need more memory (e.g. 1GB), start as follows: <p>java -Xmx1G -jar weka.jar</p> </li> <li>Open the package manager (Under 'Tools')</li> <li>Select package OpenmlWeka and click install. Afterwards, restart WEKA.</li> <li>From the Tools menu, open the 'OpenML Experimenter'.</li> </ul>"},{"location":"ecosystem/Weka/#graphical-interface","title":"Graphical Interface","text":"<p>You can solve OpenML Tasks in the Weka Experimenter, and automatically upload your experiments to OpenML (or store them locally).  </p> <ul> <li>From the Tools menu, open the 'OpenML Experimenter'.</li> <li>Enter your API key in the top field (log in first). You can also store this in a config file (see below).</li> <li>In the 'Tasks' panel, click the 'Add New' button to add new tasks. Insert the task id's as comma-separated values (e.g., '1,2,3,4,5'). Use the search function on OpenML to find interesting tasks and click the ID icon to list the ID's. In the future this search will also be integrated in WEKA.</li> <li>Add algorithms in the \"Algorithm\" panel.</li> <li>Go to the \"Run\" tab, and click on the \"Start\" button.</li> <li>The experiment will be executed and sent to OpenML.org.</li> <li>The runs will now appear on OpenML.org. You can follow their progress and check for errors on your profile page under 'Runs'.</li> </ul>"},{"location":"ecosystem/Weka/#commandline-interface","title":"CommandLine Interface","text":"<p>The Command Line interface is useful for running experiments automatically on a server, without using a GUI.</p> <ul> <li>Create a config file called <code>openml.conf</code> in a new directory called <code>.openml</code> in your home dir. It should contain the following line: <p>api_key = YOUR_KEY</p> </li> <li>Execute the following command: <p>java -cp weka.jar openml.experiment.TaskBasedExperiment -T  -C  --  <li>For example, the following command will run Weka's J48 algorithm on Task 1: <p>java -cp OpenWeka.beta.jar openml.experiment.TaskBasedExperiment -T 1 -C weka.classifiers.trees.J48</p> </li> <li>The following suffix will set some parameters of this classifier: <p>-- -C 0.25 -M 2</p> </li>"},{"location":"ecosystem/Weka/#api-reference","title":"API reference","text":"<p>Check the Weka integration Java Docs for more details about the possibilities.</p>"},{"location":"ecosystem/Weka/#issues","title":"Issues","text":"<p>Please report any bugs that you may encounter in the issue tracker: https://github.com/openml/openml-weka Or email to j.n.van.rijn@liacs.leidenuniv.nl</p>"},{"location":"ecosystem/mlr/","title":"Machine Learning in R (mlr)","text":"<p>OpenML is readily integrated with mlr through the mlr3oml package.</p> <p>Example</p> <pre><code>library(mlr3oml)\nlibrary(mlr3)\n\n# Search for specific datasets\nodatasets = list_oml_data(\nnumber_features = c(10, 20),\nnumber_instances = c(45000, 50000),\nnumber_classes = 2\n)\n\n# Get dataset\nodata = odt(id = 1590)\n# Access the actual data\nodata$data\n\n# Convert to an mlr3::Task\ntsk_adult = as_task(odata, target = \"class\")\n</code></pre> <p>Key features:  </p> <ul> <li>Query and download OpenML datasets and use them however you like  </li> <li>Build any mlr learner, run it on any task and save the experiment as run objects  </li> <li>Upload your runs for collaboration or publishing  </li> <li>Query, download and reuse all shared runs  </li> </ul> <p>There is also an older (deprecated) OpenML R package.</p>"},{"location":"ecosystem/Scikit-learn/","title":"scikit-learn","text":"<p>OpenML is readily integrated with scikit-learn through the Python API. This page provides a brief overview of the key features and installation instructions. For more detailed API documentation, please refer to the official documentation.</p>"},{"location":"ecosystem/Scikit-learn/#key-features","title":"Key features:","text":"<ul> <li>Query and download OpenML datasets and use them however you like</li> <li>Build any sklearn estimator or pipeline and convert to OpenML flows</li> <li>Run any flow on any task and save the experiment as run objects</li> <li>Upload your runs for collaboration or publishing</li> <li>Query, download and reuse all shared runs</li> </ul>"},{"location":"ecosystem/Scikit-learn/#installation","title":"Installation","text":"<pre><code>pip install openml\n</code></pre>"},{"location":"ecosystem/Scikit-learn/#query-and-download-data","title":"Query and download data","text":"<pre><code>import openml\n\n# List all datasets and their properties\nopenml.datasets.list_datasets(output_format=\"dataframe\")\n\n# Get dataset by ID\ndataset = openml.datasets.get_dataset(61)\n\n# Get dataset by name\ndataset = openml.datasets.get_dataset('Fashion-MNIST')\n\n# Get the data itself as a dataframe (or otherwise)\nX, y, _, _ = dataset.get_data(dataset_format=\"dataframe\")\n</code></pre>"},{"location":"ecosystem/Scikit-learn/#download-tasks-run-models-locally-publish-results-with-scikit-learn","title":"Download tasks, run models locally, publish results (with scikit-learn)","text":"<pre><code>from sklearn import ensemble\nfrom openml import tasks, runs\n\n# Build any model you like\nclf = ensemble.RandomForestClassifier()\n\n# Download any OpenML task\ntask = tasks.get_task(3954)\n\n# Run and evaluate your model on the task\nrun = runs.run_model_on_task(clf, task)\n\n# Share the results on OpenML. Your API key can be found in your account.\n# openml.config.apikey = 'YOUR_KEY'\nrun.publish()\n</code></pre>"},{"location":"ecosystem/Scikit-learn/#openml-benchmarks","title":"OpenML Benchmarks","text":"<pre><code># List all tasks in a benchmark\nbenchmark = openml.study.get_suite('OpenML-CC18')\ntasks.list_tasks(output_format=\"dataframe\", task_id=benchmark.tasks)\n\n# Return benchmark results\nopenml.evaluations.list_evaluations(\n    function=\"area_under_roc_curve\",\n    tasks=benchmark.tasks,\n    output_format=\"dataframe\"\n)\n</code></pre>"},{"location":"ecosystem/Scikit-learn/basic_tutorial/","title":"Basic tutorial","text":"In\u00a0[12]: Copied! <pre>from IPython.display import display, HTML, Markdown\nimport os\nimport yaml\nwith open(\"../../../mkdocs.yml\", \"r\") as f:\n    load_config = yaml.safe_load(f)\nrepo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\")\nbinder_url = load_config[\"binder_url\"]\nrelative_file_path = \"integrations/Scikit-learn/basic_tutorial.ipynb\"\ndisplay(HTML(f\"\"\"&lt;a target=\"_blank\" href=\"https://colab.research.google.com/github/{repo_url}/{relative_file_path}\"&gt;\n  &lt;img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/&gt;\n&lt;/a&gt;\"\"\"))\ndisplay(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\"))\n</pre> from IPython.display import display, HTML, Markdown import os import yaml with open(\"../../../mkdocs.yml\", \"r\") as f:     load_config = yaml.safe_load(f) repo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\") binder_url = load_config[\"binder_url\"] relative_file_path = \"integrations/Scikit-learn/basic_tutorial.ipynb\" display(HTML(f\"\"\" \"\"\")) display(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\")) In\u00a0[\u00a0]: Copied! <pre>!pip install openml\n</pre> !pip install openml In\u00a0[2]: Copied! <pre>import openml\nfrom sklearn import impute, tree, pipeline\n</pre> import openml from sklearn import impute, tree, pipeline In\u00a0[7]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <pre>/Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!\n  warnings.warn(\n</pre> In\u00a0[8]: Copied! <pre># Define a scikit-learn classifier or pipeline\nclf = pipeline.Pipeline(\n    steps=[\n        ('imputer', impute.SimpleImputer()),\n        ('estimator', tree.DecisionTreeClassifier())\n    ]\n)\n</pre>  # Define a scikit-learn classifier or pipeline clf = pipeline.Pipeline(     steps=[         ('imputer', impute.SimpleImputer()),         ('estimator', tree.DecisionTreeClassifier())     ] )  In\u00a0[9]: Copied! <pre># Download the OpenML task for the pendigits dataset with 10-fold\n# cross-validation.\ntask = openml.tasks.get_task(32)\ntask\n</pre>  # Download the OpenML task for the pendigits dataset with 10-fold # cross-validation. task = openml.tasks.get_task(32) task Out[9]: <pre>OpenML Classification Task\n==========================\nTask Type Description: https://test.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION\nTask ID..............: 32\nTask URL.............: https://test.openml.org/t/32\nEstimation Procedure.: crossvalidation\nTarget Feature.......: class\n# of Classes.........: 10\nCost Matrix..........: Available</pre> In\u00a0[11]: Copied! <pre># Run the scikit-learn model on the task.\nrun = openml.runs.run_model_on_task(clf, task)\n# Publish the experiment on OpenML (optional, requires an API key.\n# You can get your own API key by signing up to OpenML.org)\n</pre> # Run the scikit-learn model on the task. run = openml.runs.run_model_on_task(clf, task) # Publish the experiment on OpenML (optional, requires an API key. # You can get your own API key by signing up to OpenML.org)  In\u00a0[\u00a0]: Copied! <pre>run.publish()\nprint(f'View the run online: {run.openml_url}')\n</pre>  run.publish() print(f'View the run online: {run.openml_url}')"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/","title":"Datasets","text":"In\u00a0[2]: Copied! <pre>from IPython.display import display, HTML, Markdown\nimport os\nimport yaml\nwith open(\"../../../mkdocs.yml\", \"r\") as f:\n    load_config = yaml.safe_load(f)\nrepo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\")\nbinder_url = load_config[\"binder_url\"]\nrelative_file_path = \"integrations/Scikit-learn/datasets_tutorial.ipynb\"\ndisplay(HTML(f\"\"\"&lt;a target=\"_blank\" href=\"https://colab.research.google.com/github/{repo_url}/{relative_file_path}\"&gt;\n  &lt;img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/&gt;\n&lt;/a&gt;\"\"\"))\ndisplay(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\"))\n</pre> from IPython.display import display, HTML, Markdown import os import yaml with open(\"../../../mkdocs.yml\", \"r\") as f:     load_config = yaml.safe_load(f) repo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\") binder_url = load_config[\"binder_url\"] relative_file_path = \"integrations/Scikit-learn/datasets_tutorial.ipynb\" display(HTML(f\"\"\" \"\"\")) display(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\")) In\u00a0[9]: Copied! <pre>!pip install openml\n</pre> !pip install openml <pre>Requirement already satisfied: openml in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (0.14.2)\nRequirement already satisfied: scikit-learn&gt;=0.18 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (1.4.2)\nRequirement already satisfied: requests in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.31.0)\nRequirement already satisfied: liac-arff&gt;=2.4.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.5.0)\nRequirement already satisfied: numpy&gt;=1.6.2 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (1.26.4)\nRequirement already satisfied: minio in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (7.2.7)\nRequirement already satisfied: pandas&gt;=1.0.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.2.2)\nRequirement already satisfied: scipy&gt;=0.13.3 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (1.13.0)\nRequirement already satisfied: pyarrow in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (16.0.0)\nRequirement already satisfied: xmltodict in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (0.13.0)\nRequirement already satisfied: python-dateutil in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.9.0.post0)\nRequirement already satisfied: tzdata&gt;=2022.7 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from pandas&gt;=1.0.0-&gt;openml) (2024.1)\nRequirement already satisfied: pytz&gt;=2020.1 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from pandas&gt;=1.0.0-&gt;openml) (2024.1)\nRequirement already satisfied: six&gt;=1.5 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from python-dateutil-&gt;openml) (1.16.0)\nRequirement already satisfied: threadpoolctl&gt;=2.0.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from scikit-learn&gt;=0.18-&gt;openml) (3.5.0)\nRequirement already satisfied: joblib&gt;=1.2.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from scikit-learn&gt;=0.18-&gt;openml) (1.4.0)\nRequirement already satisfied: urllib3 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (2.2.1)\nRequirement already satisfied: typing-extensions in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (4.11.0)\nRequirement already satisfied: pycryptodome in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (3.20.0)\nRequirement already satisfied: certifi in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (2024.2.2)\nRequirement already satisfied: argon2-cffi in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (23.1.0)\nRequirement already satisfied: idna&lt;4,&gt;=2.5 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from requests-&gt;openml) (3.7)\nRequirement already satisfied: charset-normalizer&lt;4,&gt;=2 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from requests-&gt;openml) (3.3.2)\nRequirement already satisfied: argon2-cffi-bindings in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from argon2-cffi-&gt;minio-&gt;openml) (21.2.0)\nRequirement already satisfied: cffi&gt;=1.0.1 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from argon2-cffi-bindings-&gt;argon2-cffi-&gt;minio-&gt;openml) (1.16.0)\nRequirement already satisfied: pycparser in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from cffi&gt;=1.0.1-&gt;argon2-cffi-bindings-&gt;argon2-cffi-&gt;minio-&gt;openml) (2.22)\n\n[notice] A new release of pip is available: 23.0.1 -&gt; 24.0\n[notice] To update, run: pip install --upgrade pip\n</pre> In\u00a0[2]: Copied! <pre># License: BSD 3-Clauses\n\nimport openml\nimport pandas as pd\nfrom openml.datasets import edit_dataset, fork_dataset, get_dataset\n</pre> # License: BSD 3-Clauses  import openml import pandas as pd from openml.datasets import edit_dataset, fork_dataset, get_dataset In\u00a0[3]: Copied! <pre>datalist = openml.datasets.list_datasets(output_format=\"dataframe\")\ndatalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]\n\nprint(f\"First 10 of {len(datalist)} datasets...\")\ndatalist.head(n=10)\n\n# The same can be done with lesser lines of code\nopenml_df = openml.datasets.list_datasets(output_format=\"dataframe\")\nopenml_df.head(n=10)\n</pre> datalist = openml.datasets.list_datasets(output_format=\"dataframe\") datalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]  print(f\"First 10 of {len(datalist)} datasets...\") datalist.head(n=10)  # The same can be done with lesser lines of code openml_df = openml.datasets.list_datasets(output_format=\"dataframe\") openml_df.head(n=10) <pre>First 10 of 5466 datasets...\n</pre> Out[3]: did name version uploader status format MajorityClassSize MaxNominalAttDistinctValues MinorityClassSize NumberOfClasses NumberOfFeatures NumberOfInstances NumberOfInstancesWithMissingValues NumberOfMissingValues NumberOfNumericFeatures NumberOfSymbolicFeatures 2 2 anneal 1 1 active ARFF 684.0 7.0 8.0 5.0 39.0 898.0 898.0 22175.0 6.0 33.0 3 3 kr-vs-kp 1 1 active ARFF 1669.0 3.0 1527.0 2.0 37.0 3196.0 0.0 0.0 0.0 37.0 4 4 labor 1 1 active ARFF 37.0 3.0 20.0 2.0 17.0 57.0 56.0 326.0 8.0 9.0 5 5 arrhythmia 1 1 active ARFF 245.0 13.0 2.0 13.0 280.0 452.0 384.0 408.0 206.0 74.0 6 6 letter 1 1 active ARFF 813.0 26.0 734.0 26.0 17.0 20000.0 0.0 0.0 16.0 1.0 7 7 audiology 1 1 active ARFF 57.0 24.0 1.0 24.0 70.0 226.0 222.0 317.0 0.0 70.0 8 8 liver-disorders 1 1 active ARFF NaN NaN NaN 0.0 6.0 345.0 0.0 0.0 6.0 0.0 9 9 autos 1 1 active ARFF 67.0 22.0 3.0 6.0 26.0 205.0 46.0 59.0 15.0 11.0 10 10 lymph 1 1 active ARFF 81.0 8.0 2.0 4.0 19.0 148.0 0.0 0.0 3.0 16.0 11 11 balance-scale 1 1 active ARFF 288.0 3.0 49.0 3.0 5.0 625.0 0.0 0.0 4.0 1.0 In\u00a0[4]: Copied! <pre>datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20)\n\"\"\ndatalist.query('name == \"eeg-eye-state\"')\n\"\"\ndatalist.query(\"NumberOfClasses &gt; 50\")\n</pre> datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20) \"\" datalist.query('name == \"eeg-eye-state\"') \"\" datalist.query(\"NumberOfClasses &gt; 50\") Out[4]: did name NumberOfInstances NumberOfFeatures NumberOfClasses 1491 1491 one-hundred-plants-margin 1600.0 65.0 100.0 1492 1492 one-hundred-plants-shape 1600.0 65.0 100.0 1493 1493 one-hundred-plants-texture 1599.0 65.0 100.0 4552 4552 BachChoralHarmony 5665.0 17.0 102.0 41167 41167 dionis 416188.0 61.0 355.0 41169 41169 helena 65196.0 28.0 100.0 41960 41960 seattlecrime6 523590.0 8.0 144.0 41983 41983 CIFAR-100 60000.0 3073.0 100.0 42078 42078 beer_reviews 1586614.0 13.0 104.0 42087 42087 beer_reviews 1586614.0 13.0 104.0 42088 42088 beer_reviews 1586614.0 13.0 104.0 42089 42089 vancouver_employee 1586614.0 13.0 104.0 42123 42123 article_influence 3615.0 7.0 3169.0 42223 42223 dataset-autoHorse_fixed 201.0 69.0 186.0 42396 42396 aloi 108000.0 129.0 1000.0 43723 43723 Toronto-Apartment-Rental-Price 1124.0 7.0 188.0 44282 44282 Meta_Album_PLK_Mini 3440.0 3.0 86.0 44283 44283 Meta_Album_FLW_Mini 4080.0 3.0 102.0 44284 44284 Meta_Album_SPT_Mini 2920.0 3.0 73.0 44285 44285 Meta_Album_BRD_Mini 12600.0 3.0 315.0 44288 44288 Meta_Album_TEX_Mini 2560.0 3.0 64.0 44289 44289 Meta_Album_CRS_Mini 7840.0 3.0 196.0 44292 44292 Meta_Album_INS_2_Mini 4080.0 3.0 102.0 44298 44298 Meta_Album_DOG_Mini 4800.0 3.0 120.0 44304 44304 Meta_Album_TEX_ALOT_Mini 10000.0 3.0 250.0 44306 44306 Meta_Album_INS_Mini 4160.0 3.0 104.0 44317 44317 Meta_Album_PLK_Extended 473273.0 3.0 102.0 44318 44318 Meta_Album_FLW_Extended 8189.0 3.0 102.0 44319 44319 Meta_Album_SPT_Extended 10416.0 3.0 73.0 44320 44320 Meta_Album_BRD_Extended 49054.0 3.0 315.0 44322 44322 Meta_Album_TEX_Extended 8675.0 3.0 64.0 44323 44323 Meta_Album_CRS_Extended 16185.0 3.0 196.0 44326 44326 Meta_Album_INS_2_Extended 75222.0 3.0 102.0 44331 44331 Meta_Album_DOG_Extended 20480.0 3.0 120.0 44337 44337 Meta_Album_TEX_ALOT_Extended 25000.0 3.0 250.0 44340 44340 Meta_Album_INS_Extended 170506.0 3.0 117.0 44533 44533 dionis_seed_0_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44534 44534 dionis_seed_1_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44535 44535 dionis_seed_2_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44536 44536 dionis_seed_3_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44537 44537 dionis_seed_4_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44728 44728 helena_seed_0_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44729 44729 helena_seed_1_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44730 44730 helena_seed_2_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44731 44731 helena_seed_3_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44732 44732 helena_seed_4_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 45049 45049 MD_MIX_Mini_Copy 28240.0 69.0 706.0 45102 45102 dailybike 731.0 13.0 606.0 45103 45103 dailybike 731.0 13.0 606.0 45104 45104 PLK_Mini_Copy 3440.0 3.0 86.0 45274 45274 PASS 1439588.0 7.0 94137.0 45569 45569 DBLP-QuAD 10000.0 10.0 9999.0 45923 45923 IndoorScenes 15620.0 3.0 67.0 45936 45936 IndoorScenes 15620.0 3.0 67.0 In\u00a0[5]: Copied! <pre># This is done based on the dataset ID.\ndataset = openml.datasets.get_dataset(1471)\n\n# Print a summary\nprint(\n    f\"This is dataset '{dataset.name}', the target feature is \"\n    f\"'{dataset.default_target_attribute}'\"\n)\nprint(f\"URL: {dataset.url}\")\nprint(dataset.description[:500])\n</pre> # This is done based on the dataset ID. dataset = openml.datasets.get_dataset(1471)  # Print a summary print(     f\"This is dataset '{dataset.name}', the target feature is \"     f\"'{dataset.default_target_attribute}'\" ) print(f\"URL: {dataset.url}\") print(dataset.description[:500]) <pre>This is dataset 'eeg-eye-state', the target feature is 'Class'\nURL: https://api.openml.org/data/v1/download/1587924/eeg-eye-state.arff\n**Author**: Oliver Roesler  \n**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/EEG+Eye+State), Baden-Wuerttemberg, Cooperative State University (DHBW), Stuttgart, Germany  \n**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)  \n\nAll data is from one continuous EEG measurement with the Emotiv EEG Neuroheadset. The duration of the measurement was 117 seconds. The eye state was detected via a camera during the EEG measurement and added later manually to the file after\n</pre> <p>Get the actual data.</p> <p>openml-python returns data as pandas dataframes (stored in the <code>eeg</code> variable below), and also some additional metadata that we don't care about right now.</p> In\u00a0[6]: Copied! <pre>eeg, *_ = dataset.get_data()\n</pre> eeg, *_ = dataset.get_data() <p>You can optionally choose to have openml separate out a column from the dataset. In particular, many datasets for supervised problems have a set <code>default_target_attribute</code> which may help identify the target variable.</p> In\u00a0[7]: Copied! <pre>X, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nprint(X.head())\nprint(X.info())\n</pre> X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) print(X.head()) print(X.info()) <pre>        V1       V2       V3       V4       V5       V6       V7       V8  \\\n0  4329.23  4009.23  4289.23  4148.21  4350.26  4586.15  4096.92  4641.03   \n1  4324.62  4004.62  4293.85  4148.72  4342.05  4586.67  4097.44  4638.97   \n2  4327.69  4006.67  4295.38  4156.41  4336.92  4583.59  4096.92  4630.26   \n3  4328.72  4011.79  4296.41  4155.90  4343.59  4582.56  4097.44  4630.77   \n4  4326.15  4011.79  4292.31  4151.28  4347.69  4586.67  4095.90  4627.69   \n\n        V9      V10      V11      V12      V13      V14  \n0  4222.05  4238.46  4211.28  4280.51  4635.90  4393.85  \n1  4210.77  4226.67  4207.69  4279.49  4632.82  4384.10  \n2  4207.69  4222.05  4206.67  4282.05  4628.72  4389.23  \n3  4217.44  4235.38  4210.77  4287.69  4632.31  4396.41  \n4  4210.77  4244.10  4212.82  4288.21  4632.82  4398.46  \n&lt;class 'pandas.core.frame.DataFrame'&gt;\nRangeIndex: 14980 entries, 0 to 14979\nData columns (total 14 columns):\n #   Column  Non-Null Count  Dtype  \n---  ------  --------------  -----  \n 0   V1      14980 non-null  float64\n 1   V2      14980 non-null  float64\n 2   V3      14980 non-null  float64\n 3   V4      14980 non-null  float64\n 4   V5      14980 non-null  float64\n 5   V6      14980 non-null  float64\n 6   V7      14980 non-null  float64\n 7   V8      14980 non-null  float64\n 8   V9      14980 non-null  float64\n 9   V10     14980 non-null  float64\n 10  V11     14980 non-null  float64\n 11  V12     14980 non-null  float64\n 12  V13     14980 non-null  float64\n 13  V14     14980 non-null  float64\ndtypes: float64(14)\nmemory usage: 1.6 MB\nNone\n</pre> <p>Sometimes you only need access to a dataset's metadata. In those cases, you can download the dataset without downloading the data file. The dataset object can be used as normal. Whenever you use any functionality that requires the data, such as <code>get_data</code>, the data will be downloaded. Starting from 0.15, not downloading data will be the default behavior instead. The data will be downloading automatically when you try to access it through openml objects, e.g., using <code>dataset.features</code>.</p> In\u00a0[8]: Copied! <pre>dataset = openml.datasets.get_dataset(1471, download_data=False)\n</pre> dataset = openml.datasets.get_dataset(1471, download_data=False) In\u00a0[9]: Copied! <pre>eegs = eeg.sample(n=1000)\n_ = pd.plotting.scatter_matrix(\n    X.iloc[:100, :4],\n    c=y[:100],\n    figsize=(10, 10),\n    marker=\"o\",\n    hist_kwds={\"bins\": 20},\n    alpha=0.8,\n    cmap=\"plasma\",\n)\n</pre> eegs = eeg.sample(n=1000) _ = pd.plotting.scatter_matrix(     X.iloc[:100, :4],     c=y[:100],     figsize=(10, 10),     marker=\"o\",     hist_kwds={\"bins\": 20},     alpha=0.8,     cmap=\"plasma\", ) <pre>/Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages/pandas/plotting/_matplotlib/misc.py:97: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored\n  ax.scatter(\n</pre> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>Edit non-critical fields, allowed for all authorized users: description, creator, contributor, collection_date, language, citation, original_data_url, paper_url</p> In\u00a0[\u00a0]: Copied! <pre>desc = (\n    \"This data sets consists of 3 different types of irises' \"\n    \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"\n    \" stored in a 150x4 numpy.ndarray\"\n)\ndid = 128\ndata_id = edit_dataset(\n    did,\n    description=desc,\n    creator=\"R.A.Fisher\",\n    collection_date=\"1937\",\n    citation=\"The use of multiple measurements in taxonomic problems\",\n    language=\"English\",\n)\nedited_dataset = get_dataset(data_id)\nprint(f\"Edited dataset ID: {data_id}\")\n</pre> desc = (     \"This data sets consists of 3 different types of irises' \"     \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"     \" stored in a 150x4 numpy.ndarray\" ) did = 128 data_id = edit_dataset(     did,     description=desc,     creator=\"R.A.Fisher\",     collection_date=\"1937\",     citation=\"The use of multiple measurements in taxonomic problems\",     language=\"English\", ) edited_dataset = get_dataset(data_id) print(f\"Edited dataset ID: {data_id}\") <p>Editing critical fields (default_target_attribute, row_id_attribute, ignore_attribute) is allowed only for the dataset owner. Further, critical fields cannot be edited if the dataset has any tasks associated with it. To edit critical fields of a dataset (without tasks) owned by you, configure the API key: openml.config.apikey = 'FILL_IN_OPENML_API_KEY' This example here only shows a failure when trying to work on a dataset not owned by you:</p> In\u00a0[\u00a0]: Copied! <pre>try:\n    data_id = edit_dataset(1, default_target_attribute=\"shape\")\nexcept openml.exceptions.OpenMLServerException as e:\n    print(e)\n</pre> try:     data_id = edit_dataset(1, default_target_attribute=\"shape\") except openml.exceptions.OpenMLServerException as e:     print(e) In\u00a0[\u00a0]: Copied! <pre>data_id = fork_dataset(1)\nprint(data_id)\ndata_id = edit_dataset(data_id, default_target_attribute=\"shape\")\nprint(f\"Forked dataset ID: {data_id}\")\n\nopenml.config.stop_using_configuration_for_example()\n</pre> data_id = fork_dataset(1) print(data_id) data_id = edit_dataset(data_id, default_target_attribute=\"shape\") print(f\"Forked dataset ID: {data_id}\")  openml.config.stop_using_configuration_for_example()"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#datasets","title":"Datasets\u00b6","text":"<p>How to list and download datasets.</p>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#exercise-0","title":"Exercise 0\u00b6","text":"<ul> <li><p>List datasets</p> <ul> <li>Use the output_format parameter to select output type</li> <li>Default gives 'dict' (other option: 'dataframe', see below)</li> </ul> </li> </ul> <p>Note: list_datasets will return a pandas dataframe by default from 0.15. When using openml-python 0.14, <code>list_datasets</code> will warn you to use output_format='dataframe'.</p>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#exercise-1","title":"Exercise 1\u00b6","text":"<ul> <li>Find datasets with more than 10000 examples.</li> <li>Find a dataset called 'eeg_eye_state'.</li> <li>Find all datasets with more than 50 classes.</li> </ul>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#download-datasets","title":"Download datasets\u00b6","text":""},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#exercise-2","title":"Exercise 2\u00b6","text":"<ul> <li>Explore the data visually.</li> </ul>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#edit-a-created-dataset","title":"Edit a created dataset\u00b6","text":"<p>This example uses the test server, to avoid editing a dataset on the main server.</p> Warning<p>.. include:: ../../test_server_usage_warning.txt</p>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#fork-dataset","title":"Fork dataset\u00b6","text":"<p>Used to create a copy of the dataset with you as the owner. Use this API only if you are unable to edit the critical fields (default_target_attribute, ignore_attribute, row_id_attribute) of a dataset through the edit_dataset API. After the dataset is forked, you can edit the new version of the dataset using edit_dataset.</p>"},{"location":"examples/SUMMARY/","title":"SUMMARY","text":"<ul> <li>Basic<ul> <li>introduction_tutorial.py</li> <li>simple_datasets_tutorial.py</li> <li>simple_flows_and_runs_tutorial.py</li> <li>simple_suites_tutorial.py</li> </ul> </li> <li>Extended<ul> <li>configure_logging.py</li> <li>create_upload_tutorial.py</li> <li>custom_flow_.py</li> <li>datasets_tutorial.py</li> <li>fetch_evaluations_tutorial.py</li> <li>fetch_runtimes_tutorial.py</li> <li>flow_id_tutorial.py</li> <li>flows_and_runs_tutorial.py</li> <li>plot_svm_hyperparameters_tutorial.py</li> <li>run_setup_tutorial.py</li> <li>study_tutorial.py</li> <li>suites_tutorial.py</li> <li>task_manual_iteration_tutorial.py</li> <li>tasks_tutorial.py</li> </ul> </li> <li>Paper<ul> <li>2015_neurips_feurer_example.py</li> <li>2018_ida_strang_example.py</li> <li>2018_kdd_rijn_example.py</li> <li>2018_neurips_perrone_example.py</li> </ul> </li> </ul>"},{"location":"examples/20_basic/introduction_tutorial/","title":"Introduction tutorial &amp; Setup","text":"<p>OpenML is an online collaboration platform for machine learning which allows you to:</p> <ul> <li>Find or share interesting, well-documented datasets</li> <li>Define research / modelling goals (tasks)</li> <li>Explore large amounts of machine learning algorithms, with APIs in Java, R, Python</li> <li>Log and share reproducible experiments, models, results</li> <li>Works seamlessly with scikit-learn and other libraries</li> <li>Large scale benchmarking, compare to state of the art</li> </ul> In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom sklearn import neighbors\n</pre>  import openml from sklearn import neighbors <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>When using the main server instead, make sure your apikey is configured. This can be done with the following line of code (uncomment it!). Never share your apikey with others.</p> In\u00a0[\u00a0]: Copied! <pre># openml.config.apikey = 'YOURKEY'\n</pre> # openml.config.apikey = 'YOURKEY' In\u00a0[\u00a0]: Copied! <pre># Uncomment and set your OpenML cache directory\n# import os\n# openml.config.cache_directory = os.path.expanduser('YOURDIR')\n</pre> # Uncomment and set your OpenML cache directory # import os # openml.config.cache_directory = os.path.expanduser('YOURDIR') In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(403)\ndata = openml.datasets.get_dataset(task.dataset_id)\nclf = neighbors.KNeighborsClassifier(n_neighbors=5)\nrun = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n# Publish the experiment on OpenML (optional, requires an API key).\n# For this tutorial, our configuration publishes to the test server\n# as to not crowd the main server with runs created by examples.\nmyrun = run.publish()\nprint(f\"kNN on {data.name}: {myrun.openml_url}\")\n</pre> task = openml.tasks.get_task(403) data = openml.datasets.get_dataset(task.dataset_id) clf = neighbors.KNeighborsClassifier(n_neighbors=5) run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False) # Publish the experiment on OpenML (optional, requires an API key). # For this tutorial, our configuration publishes to the test server # as to not crowd the main server with runs created by examples. myrun = run.publish() print(f\"kNN on {data.name}: {myrun.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/20_basic/introduction_tutorial/#introduction-tutorial-setup","title":"Introduction tutorial &amp; Setup\u00b6","text":"<p>An example how to set up OpenML-Python followed up by a simple example.</p>"},{"location":"examples/20_basic/introduction_tutorial/#installation","title":"Installation\u00b6","text":"<p>Installation is done via <code>pip</code>:</p> <pre>pip install openml\n</pre>"},{"location":"examples/20_basic/introduction_tutorial/#authentication","title":"Authentication\u00b6","text":"<p>The OpenML server can only be accessed by users who have signed up on the OpenML platform. If you don\u2019t have an account yet, sign up now. You will receive an API key, which will authenticate you to the server and allow you to download and upload datasets, tasks, runs and flows.</p> <ul> <li>Create an OpenML account (free) on https://www.openml.org.</li> <li>After logging in, open your account page (avatar on the top right)</li> <li>Open 'Account Settings', then 'API authentication' to find your API key.</li> </ul> <p>There are two ways to permanently authenticate:</p> <ul> <li>Use the <code>openml</code> CLI tool with <code>openml configure apikey MYKEY</code>, replacing MYKEY with your API key.</li> <li>Create a plain text file ~/.openml/config with the line 'apikey=MYKEY', replacing MYKEY with your API key. The config file must be in the directory ~/.openml/config and exist prior to importing the openml module.</li> </ul> <p>Alternatively, by running the code below and replacing 'YOURKEY' with your API key, you authenticate for the duration of the python process.</p>"},{"location":"examples/20_basic/introduction_tutorial/#caching","title":"Caching\u00b6","text":"<p>When downloading datasets, tasks, runs and flows, they will be cached to retrieve them without calling the server later. As with the API key, the cache directory can be either specified through the config file or through the API:</p> <ul> <li>Add the  line cachedir = 'MYDIR' to the config file, replacing 'MYDIR' with the path to the cache directory. By default, OpenML will use ~/.openml/cache as the cache directory.</li> <li>Run the code below, replacing 'YOURDIR' with the path to the cache directory.</li> </ul>"},{"location":"examples/20_basic/introduction_tutorial/#simple-example","title":"Simple Example\u00b6","text":"<p>Download the OpenML task for the eeg-eye-state.</p>"},{"location":"examples/20_basic/simple_datasets_tutorial/","title":"Datasets","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre>  import openml In\u00a0[\u00a0]: Copied! <pre>datasets_df = openml.datasets.list_datasets(output_format=\"dataframe\")\nprint(datasets_df.head(n=10))\n</pre> datasets_df = openml.datasets.list_datasets(output_format=\"dataframe\") print(datasets_df.head(n=10)) In\u00a0[\u00a0]: Copied! <pre># Iris dataset https://www.openml.org/d/61\ndataset = openml.datasets.get_dataset(61)\n\n# Print a summary\nprint(\n    f\"This is dataset '{dataset.name}', the target feature is \"\n    f\"'{dataset.default_target_attribute}'\"\n)\nprint(f\"URL: {dataset.url}\")\nprint(dataset.description[:500])\n</pre> # Iris dataset https://www.openml.org/d/61 dataset = openml.datasets.get_dataset(61)  # Print a summary print(     f\"This is dataset '{dataset.name}', the target feature is \"     f\"'{dataset.default_target_attribute}'\" ) print(f\"URL: {dataset.url}\") print(dataset.description[:500]) In\u00a0[\u00a0]: Copied! <pre>X, y, categorical_indicator, attribute_names = dataset.get_data(\n    dataset_format=\"dataframe\", target=dataset.default_target_attribute\n)\n</pre> X, y, categorical_indicator, attribute_names = dataset.get_data(     dataset_format=\"dataframe\", target=dataset.default_target_attribute ) <p>Visualize the dataset</p> In\u00a0[\u00a0]: Copied! <pre>import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nsns.set_style(\"darkgrid\")\n\n\ndef hide_current_axis(*args, **kwds):\n    plt.gca().set_visible(False)\n\n\n# We combine all the data so that we can map the different\n# examples to different colors according to the classes.\ncombined_data = pd.concat([X, y], axis=1)\niris_plot = sns.pairplot(combined_data, hue=\"class\")\niris_plot.map_upper(hide_current_axis)\nplt.show()\n\n# License: BSD 3-Clause\n</pre> import pandas as pd import seaborn as sns import matplotlib.pyplot as plt  sns.set_style(\"darkgrid\")   def hide_current_axis(*args, **kwds):     plt.gca().set_visible(False)   # We combine all the data so that we can map the different # examples to different colors according to the classes. combined_data = pd.concat([X, y], axis=1) iris_plot = sns.pairplot(combined_data, hue=\"class\") iris_plot.map_upper(hide_current_axis) plt.show()  # License: BSD 3-Clause"},{"location":"examples/20_basic/simple_datasets_tutorial/#datasets","title":"Datasets\u00b6","text":"<p>A basic tutorial on how to list, load and visualize datasets.</p> <p>In general, we recommend working with tasks, so that the results can be easily reproduced. Furthermore, the results can be compared to existing results at OpenML. However, for the purposes of this tutorial, we are going to work with the datasets directly.</p>"},{"location":"examples/20_basic/simple_datasets_tutorial/#list-datasets","title":"List datasets\u00b6","text":""},{"location":"examples/20_basic/simple_datasets_tutorial/#download-a-dataset","title":"Download a dataset\u00b6","text":""},{"location":"examples/20_basic/simple_datasets_tutorial/#load-a-dataset","title":"Load a dataset\u00b6","text":"<p>X - An array/dataframe where each row represents one example with the corresponding feature values.</p> <p>y - the classes for each example</p> <p>categorical_indicator - an array that indicates which feature is categorical</p> <p>attribute_names - the names of the features for the examples (X) and target feature (y)</p>"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/","title":"Flows and Runs","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom sklearn import ensemble, neighbors\n</pre> import openml from sklearn import ensemble, neighbors <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>dataset = openml.datasets.get_dataset(20)\nX, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nclf = neighbors.KNeighborsClassifier(n_neighbors=3)\nclf.fit(X, y)\n</pre> dataset = openml.datasets.get_dataset(20) X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) clf = neighbors.KNeighborsClassifier(n_neighbors=3) clf.fit(X, y) In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(119)\nclf = ensemble.RandomForestClassifier()\nrun = openml.runs.run_model_on_task(clf, task)\nprint(run)\n</pre> task = openml.tasks.get_task(119) clf = ensemble.RandomForestClassifier() run = openml.runs.run_model_on_task(clf, task) print(run) In\u00a0[\u00a0]: Copied! <pre>myrun = run.publish()\nprint(f\"Run was uploaded to {myrun.openml_url}\")\nprint(f\"The flow can be found at {myrun.flow.openml_url}\")\n</pre> myrun = run.publish() print(f\"Run was uploaded to {myrun.openml_url}\") print(f\"The flow can be found at {myrun.flow.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#flows-and-runs","title":"Flows and Runs\u00b6","text":"<p>A simple tutorial on how to train/run a model and how to upload the results.</p>"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#train-a-machine-learning-model","title":"Train a machine learning model\u00b6","text":"<p>NOTE: We are using dataset 20 from the test server: https://test.openml.org/d/20</p>"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#running-a-model-on-a-task","title":"Running a model on a task\u00b6","text":""},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#publishing-the-run","title":"Publishing the run\u00b6","text":""},{"location":"examples/20_basic/simple_suites_tutorial/","title":"Benchmark suites","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre> import openml In\u00a0[\u00a0]: Copied! <pre>suite = openml.study.get_suite(99)\nprint(suite)\n</pre> suite = openml.study.get_suite(99) print(suite) <p>The benchmark suite does not download the included tasks and datasets itself, but only contains a list of which tasks constitute the study.</p> <p>Tasks can then be accessed via</p> In\u00a0[\u00a0]: Copied! <pre>tasks = suite.tasks\nprint(tasks)\n</pre> tasks = suite.tasks print(tasks) <p>and iterated over for benchmarking. For speed reasons we only iterate over the first three tasks:</p> In\u00a0[\u00a0]: Copied! <pre>for task_id in tasks[:3]:\n    task = openml.tasks.get_task(task_id)\n    print(task)\n</pre> for task_id in tasks[:3]:     task = openml.tasks.get_task(task_id)     print(task)"},{"location":"examples/20_basic/simple_suites_tutorial/#benchmark-suites","title":"Benchmark suites\u00b6","text":"<p>This is a brief showcase of OpenML benchmark suites, which were introduced by Bischl et al. (2019). Benchmark suites standardize the datasets and splits to be used in an experiment or paper. They are fully integrated into OpenML and simplify both the sharing of the setup and the results.</p>"},{"location":"examples/20_basic/simple_suites_tutorial/#openml-cc18","title":"OpenML-CC18\u00b6","text":"<p>As an example we have a look at the OpenML-CC18, which is a suite of 72 classification datasets from OpenML which were carefully selected to be usable by many algorithms and also represent datasets commonly used in machine learning research. These are all datasets from mid-2018 that satisfy a large set of clear requirements for thorough yet practical benchmarking:</p> <ol> <li>the number of observations are between 500 and 100,000 to focus on medium-sized datasets,</li> <li>the number of features does not exceed 5,000 features to keep the runtime of the algorithms low</li> <li>the target attribute has at least two classes with no class having less than 20 observations</li> <li>the ratio of the minority class and the majority class is above 0.05 (to eliminate highly imbalanced datasets which require special treatment for both algorithms and evaluation measures).</li> </ol> <p>A full description can be found in the OpenML benchmarking docs.</p> <p>In this example we'll focus on how to use benchmark suites in practice.</p>"},{"location":"examples/20_basic/simple_suites_tutorial/#downloading-benchmark-suites","title":"Downloading benchmark suites\u00b6","text":""},{"location":"examples/20_basic/simple_suites_tutorial/#further-examples","title":"Further examples\u00b6","text":"<ul> <li>Suites Tutorial</li> <li>Study Tutoral</li> <li>Paper example: Strang et al.</li> </ul> <p>License: BSD 3-Clause</p>"},{"location":"examples/30_extended/configure_logging/","title":"Logging","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n\nopenml.datasets.get_dataset(\"iris\")\n</pre> import openml  openml.datasets.get_dataset(\"iris\") <p>With default configuration, the above example will show no output to console. However, in your cache directory you should find a file named 'openml_python.log', which has a DEBUG message written to it. It should be either like \"[DEBUG] [10:46:19:openml.datasets.dataset] Saved dataset 61: iris to file ...\" or like \"[DEBUG] [10:49:38:openml.datasets.dataset] Data pickle file already exists and is up to date.\" , depending on whether or not you had downloaded iris before. The processed log levels can be configured programmatically:</p> In\u00a0[\u00a0]: Copied! <pre>import logging\n\nopenml.config.set_console_log_level(logging.DEBUG)\nopenml.config.set_file_log_level(logging.WARNING)\nopenml.datasets.get_dataset(\"iris\")\n</pre> import logging  openml.config.set_console_log_level(logging.DEBUG) openml.config.set_file_log_level(logging.WARNING) openml.datasets.get_dataset(\"iris\") <p>Now the log level that was previously written to file should also be shown in the console. The message is now no longer written to file as the <code>file_log</code> was set to level <code>WARNING</code>.</p> <p>It is also possible to specify the desired log levels through the configuration file. This way you will not need to set them on each script separately. Add the  line verbosity = NUMBER and/or file_verbosity = NUMBER to the config file, where 'NUMBER' should be one of:</p> <ul> <li>0: <code>logging.WARNING</code> and up.</li> <li>1: <code>logging.INFO</code> and up.</li> <li>2: <code>logging.DEBUG</code> and up (i.e. all messages).</li> </ul> <p>License: BSD 3-Clause</p>"},{"location":"examples/30_extended/configure_logging/#logging","title":"Logging\u00b6","text":"<p>This tutorial explains openml-python logging, and shows how to configure it. Openml-python uses the Python logging module to provide users with log messages. Each log message is assigned a level of importance, see the table in Python's logging tutorial here.</p> <p>By default, openml-python will print log messages of level <code>WARNING</code> and above to console. All log messages (including <code>DEBUG</code> and <code>INFO</code>) are also saved in a file, which can be found in your cache directory (see also the introduction tutorial. These file logs are automatically deleted if needed, and use at most 2MB of space.</p> <p>It is possible to configure what log levels to send to console and file. When downloading a dataset from OpenML, a <code>DEBUG</code>-level message is written:</p>"},{"location":"examples/30_extended/create_upload_tutorial/","title":"Dataset upload tutorial","text":"In\u00a0[\u00a0]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport sklearn.datasets\nfrom scipy.sparse import coo_matrix\n\nimport openml\nfrom openml.datasets.functions import create_dataset\n</pre> import numpy as np import pandas as pd import sklearn.datasets from scipy.sparse import coo_matrix  import openml from openml.datasets.functions import create_dataset <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>Below we will cover the following cases of the dataset object:</p> <ul> <li>A numpy array</li> <li>A list</li> <li>A pandas dataframe</li> <li>A sparse matrix</li> <li>A pandas sparse dataframe</li> </ul> In\u00a0[\u00a0]: Copied! <pre>diabetes = sklearn.datasets.load_diabetes()\nname = \"Diabetes(scikit-learn)\"\nX = diabetes.data\ny = diabetes.target\nattribute_names = diabetes.feature_names\ndescription = diabetes.DESCR\n</pre> diabetes = sklearn.datasets.load_diabetes() name = \"Diabetes(scikit-learn)\" X = diabetes.data y = diabetes.target attribute_names = diabetes.feature_names description = diabetes.DESCR <p>OpenML does not distinguish between the attributes and targets on the data level and stores all data in a single matrix.</p> <p>The target feature is indicated as meta-data of the dataset (and tasks on that data).</p> In\u00a0[\u00a0]: Copied! <pre>data = np.concatenate((X, y.reshape((-1, 1))), axis=1)\nattribute_names = list(attribute_names)\nattributes = [(attribute_name, \"REAL\") for attribute_name in attribute_names] + [\n    (\"class\", \"INTEGER\")\n]\ncitation = (\n    \"Bradley Efron, Trevor Hastie, Iain Johnstone and \"\n    \"Robert Tibshirani (2004) (Least Angle Regression) \"\n    \"Annals of Statistics (with discussion), 407-499\"\n)\npaper_url = \"https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf\"\n</pre> data = np.concatenate((X, y.reshape((-1, 1))), axis=1) attribute_names = list(attribute_names) attributes = [(attribute_name, \"REAL\") for attribute_name in attribute_names] + [     (\"class\", \"INTEGER\") ] citation = (     \"Bradley Efron, Trevor Hastie, Iain Johnstone and \"     \"Robert Tibshirani (2004) (Least Angle Regression) \"     \"Annals of Statistics (with discussion), 407-499\" ) paper_url = \"https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf\" In\u00a0[\u00a0]: Copied! <pre>diabetes_dataset = create_dataset(\n    # The name of the dataset (needs to be unique).\n    # Must not be longer than 128 characters and only contain\n    # a-z, A-Z, 0-9 and the following special characters: _\\-\\.(),\n    name=name,\n    # Textual description of the dataset.\n    description=description,\n    # The person who created the dataset.\n    creator=\"Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani\",\n    # People who contributed to the current version of the dataset.\n    contributor=None,\n    # The date the data was originally collected, given by the uploader.\n    collection_date=\"09-01-2012\",\n    # Language in which the data is represented.\n    # Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    language=\"English\",\n    # License under which the data is/will be distributed.\n    licence=\"BSD (from scikit-learn)\",\n    # Name of the target. Can also have multiple values (comma-separated).\n    default_target_attribute=\"class\",\n    # The attribute that represents the row-id column, if present in the\n    # dataset.\n    row_id_attribute=None,\n    # Attribute or list of attributes that should be excluded in modelling, such as\n    # identifiers and indexes. E.g. \"feat1\" or [\"feat1\",\"feat2\"]\n    ignore_attribute=None,\n    # How to cite the paper.\n    citation=citation,\n    # Attributes of the data\n    attributes=attributes,\n    data=data,\n    # A version label which is provided by the user.\n    version_label=\"test\",\n    original_data_url=\"https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html\",\n    paper_url=paper_url,\n)\n</pre> diabetes_dataset = create_dataset(     # The name of the dataset (needs to be unique).     # Must not be longer than 128 characters and only contain     # a-z, A-Z, 0-9 and the following special characters: _\\-\\.(),     name=name,     # Textual description of the dataset.     description=description,     # The person who created the dataset.     creator=\"Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani\",     # People who contributed to the current version of the dataset.     contributor=None,     # The date the data was originally collected, given by the uploader.     collection_date=\"09-01-2012\",     # Language in which the data is represented.     # Starts with 1 upper case letter, rest lower case, e.g. 'English'.     language=\"English\",     # License under which the data is/will be distributed.     licence=\"BSD (from scikit-learn)\",     # Name of the target. Can also have multiple values (comma-separated).     default_target_attribute=\"class\",     # The attribute that represents the row-id column, if present in the     # dataset.     row_id_attribute=None,     # Attribute or list of attributes that should be excluded in modelling, such as     # identifiers and indexes. E.g. \"feat1\" or [\"feat1\",\"feat2\"]     ignore_attribute=None,     # How to cite the paper.     citation=citation,     # Attributes of the data     attributes=attributes,     data=data,     # A version label which is provided by the user.     version_label=\"test\",     original_data_url=\"https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html\",     paper_url=paper_url, ) In\u00a0[\u00a0]: Copied! <pre>diabetes_dataset.publish()\nprint(f\"URL for dataset: {diabetes_dataset.openml_url}\")\n</pre>  diabetes_dataset.publish() print(f\"URL for dataset: {diabetes_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>data = [\n    [\"sunny\", 85, 85, \"FALSE\", \"no\"],\n    [\"sunny\", 80, 90, \"TRUE\", \"no\"],\n    [\"overcast\", 83, 86, \"FALSE\", \"yes\"],\n    [\"rainy\", 70, 96, \"FALSE\", \"yes\"],\n    [\"rainy\", 68, 80, \"FALSE\", \"yes\"],\n    [\"rainy\", 65, 70, \"TRUE\", \"no\"],\n    [\"overcast\", 64, 65, \"TRUE\", \"yes\"],\n    [\"sunny\", 72, 95, \"FALSE\", \"no\"],\n    [\"sunny\", 69, 70, \"FALSE\", \"yes\"],\n    [\"rainy\", 75, 80, \"FALSE\", \"yes\"],\n    [\"sunny\", 75, 70, \"TRUE\", \"yes\"],\n    [\"overcast\", 72, 90, \"TRUE\", \"yes\"],\n    [\"overcast\", 81, 75, \"FALSE\", \"yes\"],\n    [\"rainy\", 71, 91, \"TRUE\", \"no\"],\n]\n\nattribute_names = [\n    (\"outlook\", [\"sunny\", \"overcast\", \"rainy\"]),\n    (\"temperature\", \"REAL\"),\n    (\"humidity\", \"REAL\"),\n    (\"windy\", [\"TRUE\", \"FALSE\"]),\n    (\"play\", [\"yes\", \"no\"]),\n]\n\ndescription = (\n    \"The weather problem is a tiny dataset that we will use repeatedly\"\n    \" to illustrate machine learning methods. Entirely fictitious, it \"\n    \"supposedly concerns the conditions that are suitable for playing \"\n    \"some unspecified game. In general, instances in a dataset are \"\n    \"characterized by the values of features, or attributes, that measure \"\n    \"different aspects of the instance. In this case there are four \"\n    \"attributes: outlook, temperature, humidity, and windy. \"\n    \"The outcome is whether to play or not.\"\n)\n\ncitation = (\n    \"I. H. Witten, E. Frank, M. A. Hall, and ITPro,\"\n    \"Data mining practical machine learning tools and techniques, \"\n    \"third edition. Burlington, Mass.: Morgan Kaufmann Publishers, 2011\"\n)\n\nweather_dataset = create_dataset(\n    name=\"Weather\",\n    description=description,\n    creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",\n    contributor=None,\n    collection_date=\"01-01-2011\",\n    language=\"English\",\n    licence=None,\n    default_target_attribute=\"play\",\n    row_id_attribute=None,\n    ignore_attribute=None,\n    citation=citation,\n    attributes=attribute_names,\n    data=data,\n    version_label=\"example\",\n)\n</pre> data = [     [\"sunny\", 85, 85, \"FALSE\", \"no\"],     [\"sunny\", 80, 90, \"TRUE\", \"no\"],     [\"overcast\", 83, 86, \"FALSE\", \"yes\"],     [\"rainy\", 70, 96, \"FALSE\", \"yes\"],     [\"rainy\", 68, 80, \"FALSE\", \"yes\"],     [\"rainy\", 65, 70, \"TRUE\", \"no\"],     [\"overcast\", 64, 65, \"TRUE\", \"yes\"],     [\"sunny\", 72, 95, \"FALSE\", \"no\"],     [\"sunny\", 69, 70, \"FALSE\", \"yes\"],     [\"rainy\", 75, 80, \"FALSE\", \"yes\"],     [\"sunny\", 75, 70, \"TRUE\", \"yes\"],     [\"overcast\", 72, 90, \"TRUE\", \"yes\"],     [\"overcast\", 81, 75, \"FALSE\", \"yes\"],     [\"rainy\", 71, 91, \"TRUE\", \"no\"], ]  attribute_names = [     (\"outlook\", [\"sunny\", \"overcast\", \"rainy\"]),     (\"temperature\", \"REAL\"),     (\"humidity\", \"REAL\"),     (\"windy\", [\"TRUE\", \"FALSE\"]),     (\"play\", [\"yes\", \"no\"]), ]  description = (     \"The weather problem is a tiny dataset that we will use repeatedly\"     \" to illustrate machine learning methods. Entirely fictitious, it \"     \"supposedly concerns the conditions that are suitable for playing \"     \"some unspecified game. In general, instances in a dataset are \"     \"characterized by the values of features, or attributes, that measure \"     \"different aspects of the instance. In this case there are four \"     \"attributes: outlook, temperature, humidity, and windy. \"     \"The outcome is whether to play or not.\" )  citation = (     \"I. H. Witten, E. Frank, M. A. Hall, and ITPro,\"     \"Data mining practical machine learning tools and techniques, \"     \"third edition. Burlington, Mass.: Morgan Kaufmann Publishers, 2011\" )  weather_dataset = create_dataset(     name=\"Weather\",     description=description,     creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",     contributor=None,     collection_date=\"01-01-2011\",     language=\"English\",     licence=None,     default_target_attribute=\"play\",     row_id_attribute=None,     ignore_attribute=None,     citation=citation,     attributes=attribute_names,     data=data,     version_label=\"example\", ) In\u00a0[\u00a0]: Copied! <pre>weather_dataset.publish()\nprint(f\"URL for dataset: {weather_dataset.openml_url}\")\n</pre> weather_dataset.publish() print(f\"URL for dataset: {weather_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>df = pd.DataFrame(data, columns=[col_name for col_name, _ in attribute_names])\n\n# enforce the categorical column to have a categorical dtype\ndf[\"outlook\"] = df[\"outlook\"].astype(\"category\")\ndf[\"windy\"] = df[\"windy\"].astype(\"bool\")\ndf[\"play\"] = df[\"play\"].astype(\"category\")\nprint(df.info())\n</pre> df = pd.DataFrame(data, columns=[col_name for col_name, _ in attribute_names])  # enforce the categorical column to have a categorical dtype df[\"outlook\"] = df[\"outlook\"].astype(\"category\") df[\"windy\"] = df[\"windy\"].astype(\"bool\") df[\"play\"] = df[\"play\"].astype(\"category\") print(df.info()) <p>We enforce the column 'outlook' and 'play' to be a categorical dtype while the column 'windy' is kept as a boolean column. 'temperature' and 'humidity' are kept as numeric columns. Then, we can call :func:<code>openml.datasets.create_dataset</code> by passing the dataframe and fixing the parameter <code>attributes</code> to <code>'auto'</code>.</p> In\u00a0[\u00a0]: Copied! <pre>weather_dataset = create_dataset(\n    name=\"Weather\",\n    description=description,\n    creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",\n    contributor=None,\n    collection_date=\"01-01-2011\",\n    language=\"English\",\n    licence=None,\n    default_target_attribute=\"play\",\n    row_id_attribute=None,\n    ignore_attribute=None,\n    citation=citation,\n    attributes=\"auto\",\n    data=df,\n    version_label=\"example\",\n)\n</pre> weather_dataset = create_dataset(     name=\"Weather\",     description=description,     creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",     contributor=None,     collection_date=\"01-01-2011\",     language=\"English\",     licence=None,     default_target_attribute=\"play\",     row_id_attribute=None,     ignore_attribute=None,     citation=citation,     attributes=\"auto\",     data=df,     version_label=\"example\", ) In\u00a0[\u00a0]: Copied! <pre>weather_dataset.publish()\nprint(f\"URL for dataset: {weather_dataset.openml_url}\")\n</pre> weather_dataset.publish() print(f\"URL for dataset: {weather_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>sparse_data = coo_matrix(\n    ([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))\n)\n\ncolumn_names = [\n    (\"input1\", \"REAL\"),\n    (\"input2\", \"REAL\"),\n    (\"y\", \"REAL\"),\n]\n\nxor_dataset = create_dataset(\n    name=\"XOR\",\n    description=\"Dataset representing the XOR operation\",\n    creator=None,\n    contributor=None,\n    collection_date=None,\n    language=\"English\",\n    licence=None,\n    default_target_attribute=\"y\",\n    row_id_attribute=None,\n    ignore_attribute=None,\n    citation=None,\n    attributes=column_names,\n    data=sparse_data,\n    version_label=\"example\",\n)\n</pre> sparse_data = coo_matrix(     ([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1])) )  column_names = [     (\"input1\", \"REAL\"),     (\"input2\", \"REAL\"),     (\"y\", \"REAL\"), ]  xor_dataset = create_dataset(     name=\"XOR\",     description=\"Dataset representing the XOR operation\",     creator=None,     contributor=None,     collection_date=None,     language=\"English\",     licence=None,     default_target_attribute=\"y\",     row_id_attribute=None,     ignore_attribute=None,     citation=None,     attributes=column_names,     data=sparse_data,     version_label=\"example\", ) In\u00a0[\u00a0]: Copied! <pre>xor_dataset.publish()\nprint(f\"URL for dataset: {xor_dataset.openml_url}\")\n</pre> xor_dataset.publish() print(f\"URL for dataset: {xor_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>xor_dataset.publish()\nprint(f\"URL for dataset: {xor_dataset.openml_url}\")\n</pre>  xor_dataset.publish() print(f\"URL for dataset: {xor_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-upload-tutorial","title":"Dataset upload tutorial\u00b6","text":"<p>A tutorial on how to create and upload a dataset to OpenML.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-numpy-array","title":"Dataset is a numpy array\u00b6","text":"<p>A numpy array can contain lists in the case of dense data or it can contain OrderedDicts in the case of sparse data.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#prepare-dataset","title":"Prepare dataset\u00b6","text":"<p>Load an example dataset from scikit-learn which we will upload to OpenML.org via the API.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#create-the-dataset-object","title":"Create the dataset object\u00b6","text":"<p>The definition of all fields can be found in the XSD files describing the expected format:</p> <p>https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.data.upload.xsd</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-list","title":"Dataset is a list\u00b6","text":"<p>A list can contain lists in the case of dense data or it can contain OrderedDicts in the case of sparse data.</p> <p>Weather dataset: https://storm.cis.fordham.edu/~gweiss/data-mining/datasets.html</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-pandas-dataframe","title":"Dataset is a pandas DataFrame\u00b6","text":"<p>It might happen that your dataset is made of heterogeneous data which can usually be stored as a Pandas DataFrame. DataFrames offer the advantage of storing the type of data for each column as well as the attribute names. Therefore, when providing a Pandas DataFrame, OpenML can infer this information without needing to explicitly provide it when calling the function :func:<code>openml.datasets.create_dataset</code>. In this regard, you only need to pass <code>'auto'</code> to the <code>attributes</code> parameter.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-sparse-matrix","title":"Dataset is a sparse matrix\u00b6","text":""},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-pandas-dataframe-with-sparse-columns","title":"Dataset is a pandas dataframe with sparse columns\u00b6","text":"<p>sparse_data = coo_matrix( ([1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1])) ) column_names = [\"input1\", \"input2\", \"y\"] df = pd.DataFrame.sparse.from_spmatrix(sparse_data, columns=column_names) print(df.info())</p> <p>xor_dataset = create_dataset( name=\"XOR\", description=\"Dataset representing the XOR operation\", creator=None, contributor=None, collection_date=None, language=\"English\", licence=None, default_target_attribute=\"y\", row_id_attribute=None, ignore_attribute=None, citation=None, attributes=\"auto\", data=df, version_label=\"example\", )</p>"},{"location":"examples/30_extended/custom_flow_/","title":"Creating and Using a Custom Flow","text":"In\u00a0[\u00a0]: Copied! <pre>from collections import OrderedDict\nimport numpy as np\n\nimport openml\nfrom openml import OpenMLClassificationTask\nfrom openml.runs.functions import format_prediction\n</pre> from collections import OrderedDict import numpy as np  import openml from openml import OpenMLClassificationTask from openml.runs.functions import format_prediction <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>general = dict(\n    name=\"automlbenchmark_autosklearn\",\n    description=(\n        \"Auto-sklearn as set up by the AutoML Benchmark\"\n        \"Source: https://github.com/openml/automlbenchmark/releases/tag/v0.9\"\n    ),\n    external_version=\"amlb==0.9\",\n    language=\"English\",\n    tags=[\"amlb\", \"benchmark\", \"study_218\"],\n    dependencies=\"amlb==0.9\",\n)\n</pre> general = dict(     name=\"automlbenchmark_autosklearn\",     description=(         \"Auto-sklearn as set up by the AutoML Benchmark\"         \"Source: https://github.com/openml/automlbenchmark/releases/tag/v0.9\"     ),     external_version=\"amlb==0.9\",     language=\"English\",     tags=[\"amlb\", \"benchmark\", \"study_218\"],     dependencies=\"amlb==0.9\", ) <p>Next we define the flow hyperparameters. We define their name and default value in <code>parameters</code>, and provide meta-data for each hyperparameter through <code>parameters_meta_info</code>. Note that even though the argument name is <code>parameters</code> they describe the hyperparameters. The use of ordered dicts is required.</p> In\u00a0[\u00a0]: Copied! <pre>flow_hyperparameters = dict(\n    parameters=OrderedDict(time=\"240\", memory=\"32\", cores=\"8\"),\n    parameters_meta_info=OrderedDict(\n        cores=OrderedDict(description=\"number of available cores\", data_type=\"int\"),\n        memory=OrderedDict(description=\"memory in gigabytes\", data_type=\"int\"),\n        time=OrderedDict(description=\"time in minutes\", data_type=\"int\"),\n    ),\n)\n</pre> flow_hyperparameters = dict(     parameters=OrderedDict(time=\"240\", memory=\"32\", cores=\"8\"),     parameters_meta_info=OrderedDict(         cores=OrderedDict(description=\"number of available cores\", data_type=\"int\"),         memory=OrderedDict(description=\"memory in gigabytes\", data_type=\"int\"),         time=OrderedDict(description=\"time in minutes\", data_type=\"int\"),     ), ) <p>It is possible to build a flow which uses other flows. For example, the Random Forest Classifier is a flow, but you could also construct a flow which uses a Random Forest Classifier in a ML pipeline. When constructing the pipeline flow, you can use the Random Forest Classifier flow as a subflow. It allows for all hyperparameters of the Random Classifier Flow to also be specified in your pipeline flow.</p> <p>Note: you can currently only specific one subflow as part of the components.</p> <p>In this example, the auto-sklearn flow is a subflow: the auto-sklearn flow is entirely executed as part of this flow. This allows people to specify auto-sklearn hyperparameters used in this flow. In general, using a subflow is not required.</p> <p>Note: flow 9313 is not actually the right flow on the test server, but that does not matter for this demonstration.</p> In\u00a0[\u00a0]: Copied! <pre>autosklearn_flow = openml.flows.get_flow(9313)  # auto-sklearn 0.5.1\nsubflow = dict(\n    components=OrderedDict(automl_tool=autosklearn_flow),\n    # If you do not want to reference a subflow, you can use the following:\n    # components=OrderedDict(),\n)\n</pre> autosklearn_flow = openml.flows.get_flow(9313)  # auto-sklearn 0.5.1 subflow = dict(     components=OrderedDict(automl_tool=autosklearn_flow),     # If you do not want to reference a subflow, you can use the following:     # components=OrderedDict(), ) <p>With all parameters of the flow defined, we can now initialize the OpenMLFlow and publish. Because we provided all the details already, we do not need to provide a <code>model</code> to the flow.</p> <p>In our case, we don't even have a model. It is possible to have a model but still require to follow these steps when the model (python object) does not have an extensions from which to automatically extract the hyperparameters. So whether you have a model with no extension or no model at all, explicitly set the model of the flow to <code>None</code>.</p> In\u00a0[\u00a0]: Copied! <pre>autosklearn_amlb_flow = openml.flows.OpenMLFlow(\n    **general,\n    **flow_hyperparameters,\n    **subflow,\n    model=None,\n)\nautosklearn_amlb_flow.publish()\nprint(f\"autosklearn flow created: {autosklearn_amlb_flow.flow_id}\")\n</pre> autosklearn_amlb_flow = openml.flows.OpenMLFlow(     **general,     **flow_hyperparameters,     **subflow,     model=None, ) autosklearn_amlb_flow.publish() print(f\"autosklearn flow created: {autosklearn_amlb_flow.flow_id}\") In\u00a0[\u00a0]: Copied! <pre>flow_id = autosklearn_amlb_flow.flow_id\n\nparameters = [\n    OrderedDict([(\"oml:name\", \"cores\"), (\"oml:value\", 4), (\"oml:component\", flow_id)]),\n    OrderedDict([(\"oml:name\", \"memory\"), (\"oml:value\", 16), (\"oml:component\", flow_id)]),\n    OrderedDict([(\"oml:name\", \"time\"), (\"oml:value\", 120), (\"oml:component\", flow_id)]),\n]\n\ntask_id = 1200  # Iris Task\ntask = openml.tasks.get_task(task_id)\ndataset_id = task.get_dataset().dataset_id\n</pre> flow_id = autosklearn_amlb_flow.flow_id  parameters = [     OrderedDict([(\"oml:name\", \"cores\"), (\"oml:value\", 4), (\"oml:component\", flow_id)]),     OrderedDict([(\"oml:name\", \"memory\"), (\"oml:value\", 16), (\"oml:component\", flow_id)]),     OrderedDict([(\"oml:name\", \"time\"), (\"oml:value\", 120), (\"oml:component\", flow_id)]), ]  task_id = 1200  # Iris Task task = openml.tasks.get_task(task_id) dataset_id = task.get_dataset().dataset_id <p>The last bit of information for the run we need are the predicted values. The exact format of the predictions will depend on the task.</p> <p>The predictions should always be a list of lists, each list should contain:</p> <ul> <li>the repeat number: for repeated evaluation strategies. (e.g. repeated cross-validation)</li> <li>the fold number: for cross-validation. (what should this be for holdout?)</li> <li>0: this field is for backward compatibility.</li> <li>index: the row (of the original dataset) for which the prediction was made.</li> <li>p_1, ..., p_c: for each class the predicted probability of the sample belonging to that class. (no elements for regression tasks) Make sure the order of these elements follows the order of <code>task.class_labels</code>.</li> <li>the predicted class/value for the sample</li> <li>the true class/value for the sample</li> </ul> <p>When using openml-python extensions (such as through <code>run_model_on_task</code>), all of this formatting is automatic. Unfortunately we can not automate this procedure for custom flows, which means a little additional effort is required.</p> <p>Here we generated some random predictions in place. You can ignore this code, or use it to better understand the formatting of the predictions.</p> <p>Find the repeats/folds for this task:</p> In\u00a0[\u00a0]: Copied! <pre>n_repeats, n_folds, _ = task.get_split_dimensions()\nall_test_indices = [\n    (repeat, fold, index)\n    for repeat in range(n_repeats)\n    for fold in range(n_folds)\n    for index in task.get_train_test_split_indices(fold, repeat)[1]\n]\n\n# random class probabilities (Iris has 150 samples and 3 classes):\nr = np.random.rand(150 * n_repeats, 3)\n# scale the random values so that the probabilities of each sample sum to 1:\ny_proba = r / r.sum(axis=1).reshape(-1, 1)\ny_pred = y_proba.argmax(axis=1)\n\nclass_map = dict(zip(range(3), task.class_labels))\n_, y_true = task.get_X_and_y()\ny_true = [class_map[y] for y in y_true]\n\n# We format the predictions with the utility function `format_prediction`.\n# It will organize the relevant data in the expected format/order.\npredictions = []\nfor where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba):\n    repeat, fold, index = where\n\n    prediction = format_prediction(\n        task=task,\n        repeat=repeat,\n        fold=fold,\n        index=index,\n        prediction=class_map[yp],\n        truth=y,\n        proba={c: pb for (c, pb) in zip(task.class_labels, proba)},\n    )\n    predictions.append(prediction)\n</pre> n_repeats, n_folds, _ = task.get_split_dimensions() all_test_indices = [     (repeat, fold, index)     for repeat in range(n_repeats)     for fold in range(n_folds)     for index in task.get_train_test_split_indices(fold, repeat)[1] ]  # random class probabilities (Iris has 150 samples and 3 classes): r = np.random.rand(150 * n_repeats, 3) # scale the random values so that the probabilities of each sample sum to 1: y_proba = r / r.sum(axis=1).reshape(-1, 1) y_pred = y_proba.argmax(axis=1)  class_map = dict(zip(range(3), task.class_labels)) _, y_true = task.get_X_and_y() y_true = [class_map[y] for y in y_true]  # We format the predictions with the utility function `format_prediction`. # It will organize the relevant data in the expected format/order. predictions = [] for where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba):     repeat, fold, index = where      prediction = format_prediction(         task=task,         repeat=repeat,         fold=fold,         index=index,         prediction=class_map[yp],         truth=y,         proba={c: pb for (c, pb) in zip(task.class_labels, proba)},     )     predictions.append(prediction) <p>Finally we can create the OpenMLRun object and upload. We use the argument setup_string because the used flow was a script.</p> In\u00a0[\u00a0]: Copied! <pre>benchmark_command = f\"python3 runbenchmark.py auto-sklearn medium -m aws -t 119\"\nmy_run = openml.runs.OpenMLRun(\n    task_id=task_id,\n    flow_id=flow_id,\n    dataset_id=dataset_id,\n    parameter_settings=parameters,\n    setup_string=benchmark_command,\n    data_content=predictions,\n    tags=[\"study_218\"],\n    description_text=\"Run generated by the Custom Flow tutorial.\",\n)\nmy_run.publish()\nprint(\"run created:\", my_run.run_id)\n</pre> benchmark_command = f\"python3 runbenchmark.py auto-sklearn medium -m aws -t 119\" my_run = openml.runs.OpenMLRun(     task_id=task_id,     flow_id=flow_id,     dataset_id=dataset_id,     parameter_settings=parameters,     setup_string=benchmark_command,     data_content=predictions,     tags=[\"study_218\"],     description_text=\"Run generated by the Custom Flow tutorial.\", ) my_run.publish() print(\"run created:\", my_run.run_id) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/custom_flow_/#creating-and-using-a-custom-flow","title":"Creating and Using a Custom Flow\u00b6","text":"<p>The most convenient way to create a flow for your machine learning workflow is to generate it automatically as described in the \"Obtaining Flow IDs\" tutorial. However, there are scenarios where this is not possible, such as when the flow uses a framework without an extension or when the flow is described by a script.</p> <p>In those cases you can still create a custom flow by following the steps of this tutorial. As an example we will use the flows generated for the AutoML Benchmark, and also show how to link runs to the custom flow.</p>"},{"location":"examples/30_extended/custom_flow_/#1-defining-the-flow","title":"1. Defining the flow\u00b6","text":"<p>The first step is to define all the hyperparameters of your flow. The API pages feature a descriptions of each variable of the :class:<code>openml.flows.OpenMLFlow</code>. Note that <code>external version</code> and <code>name</code> together uniquely identify a flow.</p> <p>The AutoML Benchmark runs AutoML systems across a range of tasks. OpenML stores Flows for each AutoML system. However, the AutoML benchmark adds preprocessing to the flow, so should be described in a new flow.</p> <p>We will break down the flow arguments into several groups, for the tutorial. First we will define the name and version information. Make sure to leave enough information so others can determine exactly which version of the package/script is used. Use tags so users can find your flow easily.</p>"},{"location":"examples/30_extended/custom_flow_/#2-using-the-flow","title":"2. Using the flow\u00b6","text":"<p>This Section will show how to upload run data for your custom flow. Take care to change the values of parameters as well as the task id, to reflect the actual run. Task and parameter values in the example are fictional.</p>"},{"location":"examples/30_extended/datasets_tutorial/","title":"Datasets","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nimport pandas as pd\nfrom openml.datasets import edit_dataset, fork_dataset, get_dataset\n</pre> import openml import pandas as pd from openml.datasets import edit_dataset, fork_dataset, get_dataset In\u00a0[\u00a0]: Copied! <pre>datalist = openml.datasets.list_datasets()\ndatalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]\n\nprint(f\"First 10 of {len(datalist)} datasets...\")\ndatalist.head(n=10)\n\n# The same can be done with lesser lines of code\nopenml_df = openml.datasets.list_datasets()\nopenml_df.head(n=10)\n</pre> datalist = openml.datasets.list_datasets() datalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]  print(f\"First 10 of {len(datalist)} datasets...\") datalist.head(n=10)  # The same can be done with lesser lines of code openml_df = openml.datasets.list_datasets() openml_df.head(n=10) In\u00a0[\u00a0]: Copied! <pre>datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20)\n</pre> datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20) In\u00a0[\u00a0]: Copied! <pre>datalist.query('name == \"eeg-eye-state\"')\n</pre> datalist.query('name == \"eeg-eye-state\"') In\u00a0[\u00a0]: Copied! <pre>datalist.query(\"NumberOfClasses &gt; 50\")\n</pre> datalist.query(\"NumberOfClasses &gt; 50\") In\u00a0[\u00a0]: Copied! <pre># This is done based on the dataset ID.\ndataset = openml.datasets.get_dataset(1471)\n\n# Print a summary\nprint(\n    f\"This is dataset '{dataset.name}', the target feature is \"\n    f\"'{dataset.default_target_attribute}'\"\n)\nprint(f\"URL: {dataset.url}\")\nprint(dataset.description[:500])\n</pre> # This is done based on the dataset ID. dataset = openml.datasets.get_dataset(1471)  # Print a summary print(     f\"This is dataset '{dataset.name}', the target feature is \"     f\"'{dataset.default_target_attribute}'\" ) print(f\"URL: {dataset.url}\") print(dataset.description[:500]) <p>Get the actual data.</p> <p>openml-python returns data as pandas dataframes (stored in the <code>eeg</code> variable below), and also some additional metadata that we don't care about right now.</p> In\u00a0[\u00a0]: Copied! <pre>eeg, *_ = dataset.get_data()\n</pre> eeg, *_ = dataset.get_data() <p>You can optionally choose to have openml separate out a column from the dataset. In particular, many datasets for supervised problems have a set <code>default_target_attribute</code> which may help identify the target variable.</p> In\u00a0[\u00a0]: Copied! <pre>X, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nprint(X.head())\nprint(X.info())\n</pre> X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) print(X.head()) print(X.info()) <p>Sometimes you only need access to a dataset's metadata. In those cases, you can download the dataset without downloading the data file. The dataset object can be used as normal. Whenever you use any functionality that requires the data, such as <code>get_data</code>, the data will be downloaded. Starting from 0.15, not downloading data will be the default behavior instead. The data will be downloading automatically when you try to access it through openml objects, e.g., using <code>dataset.features</code>.</p> In\u00a0[\u00a0]: Copied! <pre>dataset = openml.datasets.get_dataset(1471)\n</pre> dataset = openml.datasets.get_dataset(1471) In\u00a0[\u00a0]: Copied! <pre>eegs = eeg.sample(n=1000)\n_ = pd.plotting.scatter_matrix(\n    X.iloc[:100, :4],\n    c=y[:100],\n    figsize=(10, 10),\n    marker=\"o\",\n    hist_kwds={\"bins\": 20},\n    alpha=0.8,\n    cmap=\"plasma\",\n)\n</pre> eegs = eeg.sample(n=1000) _ = pd.plotting.scatter_matrix(     X.iloc[:100, :4],     c=y[:100],     figsize=(10, 10),     marker=\"o\",     hist_kwds={\"bins\": 20},     alpha=0.8,     cmap=\"plasma\", ) In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>Edit non-critical fields, allowed for all authorized users: description, creator, contributor, collection_date, language, citation, original_data_url, paper_url</p> In\u00a0[\u00a0]: Copied! <pre>desc = (\n    \"This data sets consists of 3 different types of irises' \"\n    \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"\n    \" stored in a 150x4 numpy.ndarray\"\n)\ndid = 128\ndata_id = edit_dataset(\n    did,\n    description=desc,\n    creator=\"R.A.Fisher\",\n    collection_date=\"1937\",\n    citation=\"The use of multiple measurements in taxonomic problems\",\n    language=\"English\",\n)\nedited_dataset = get_dataset(data_id)\nprint(f\"Edited dataset ID: {data_id}\")\n</pre> desc = (     \"This data sets consists of 3 different types of irises' \"     \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"     \" stored in a 150x4 numpy.ndarray\" ) did = 128 data_id = edit_dataset(     did,     description=desc,     creator=\"R.A.Fisher\",     collection_date=\"1937\",     citation=\"The use of multiple measurements in taxonomic problems\",     language=\"English\", ) edited_dataset = get_dataset(data_id) print(f\"Edited dataset ID: {data_id}\") <p>Editing critical fields (default_target_attribute, row_id_attribute, ignore_attribute) is allowed only for the dataset owner. Further, critical fields cannot be edited if the dataset has any tasks associated with it. To edit critical fields of a dataset (without tasks) owned by you, configure the API key: openml.config.apikey = 'FILL_IN_OPENML_API_KEY' This example here only shows a failure when trying to work on a dataset not owned by you:</p> In\u00a0[\u00a0]: Copied! <pre>try:\n    data_id = edit_dataset(1, default_target_attribute=\"shape\")\nexcept openml.exceptions.OpenMLServerException as e:\n    print(e)\n</pre> try:     data_id = edit_dataset(1, default_target_attribute=\"shape\") except openml.exceptions.OpenMLServerException as e:     print(e) In\u00a0[\u00a0]: Copied! <pre>data_id = fork_dataset(1)\nprint(data_id)\ndata_id = edit_dataset(data_id, default_target_attribute=\"shape\")\nprint(f\"Forked dataset ID: {data_id}\")\n</pre> data_id = fork_dataset(1) print(data_id) data_id = edit_dataset(data_id, default_target_attribute=\"shape\") print(f\"Forked dataset ID: {data_id}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clauses\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clauses"},{"location":"examples/30_extended/datasets_tutorial/#datasets","title":"Datasets\u00b6","text":"<p>How to list and download datasets.</p>"},{"location":"examples/30_extended/datasets_tutorial/#exercise-0","title":"Exercise 0\u00b6","text":"<ul> <li>List datasets</li> </ul>"},{"location":"examples/30_extended/datasets_tutorial/#exercise-1","title":"Exercise 1\u00b6","text":"<ul> <li>Find datasets with more than 10000 examples.</li> <li>Find a dataset called 'eeg_eye_state'.</li> <li>Find all datasets with more than 50 classes.</li> </ul>"},{"location":"examples/30_extended/datasets_tutorial/#download-datasets","title":"Download datasets\u00b6","text":""},{"location":"examples/30_extended/datasets_tutorial/#exercise-2","title":"Exercise 2\u00b6","text":"<ul> <li>Explore the data visually.</li> </ul>"},{"location":"examples/30_extended/datasets_tutorial/#edit-a-created-dataset","title":"Edit a created dataset\u00b6","text":"<p>This example uses the test server, to avoid editing a dataset on the main server.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p>"},{"location":"examples/30_extended/datasets_tutorial/#fork-dataset","title":"Fork dataset\u00b6","text":"<p>Used to create a copy of the dataset with you as the owner. Use this API only if you are unable to edit the critical fields (default_target_attribute, ignore_attribute, row_id_attribute) of a dataset through the edit_dataset API. After the dataset is forked, you can edit the new version of the dataset using edit_dataset.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/","title":"Fetching Evaluations","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre> import openml In\u00a0[\u00a0]: Copied! <pre>openml.evaluations.list_evaluations(\n    function=\"predictive_accuracy\", size=10\n)\n\n# Using other evaluation metrics, 'precision' in this case\nevals = openml.evaluations.list_evaluations(\n    function=\"precision\", size=10\n)\n\n# Querying the returned results for precision above 0.98\nprint(evals[evals.value &gt; 0.98])\n</pre> openml.evaluations.list_evaluations(     function=\"predictive_accuracy\", size=10 )  # Using other evaluation metrics, 'precision' in this case evals = openml.evaluations.list_evaluations(     function=\"precision\", size=10 )  # Querying the returned results for precision above 0.98 print(evals[evals.value &gt; 0.98]) In\u00a0[\u00a0]: Copied! <pre>task_id = 167140  # https://www.openml.org/t/167140\ntask = openml.tasks.get_task(task_id)\nprint(task)\n</pre> task_id = 167140  # https://www.openml.org/t/167140 task = openml.tasks.get_task(task_id) print(task) In\u00a0[\u00a0]: Copied! <pre>metric = \"predictive_accuracy\"\nevals = openml.evaluations.list_evaluations(\n    function=metric, tasks=[task_id], output_format=\"dataframe\"\n)\n# Displaying the first 10 rows\nprint(evals.head(n=10))\n# Sorting the evaluations in decreasing order of the metric chosen\nevals = evals.sort_values(by=\"value\", ascending=False)\nprint(\"\\nDisplaying head of sorted dataframe: \")\nprint(evals.head())\n</pre> metric = \"predictive_accuracy\" evals = openml.evaluations.list_evaluations(     function=metric, tasks=[task_id], output_format=\"dataframe\" ) # Displaying the first 10 rows print(evals.head(n=10)) # Sorting the evaluations in decreasing order of the metric chosen evals = evals.sort_values(by=\"value\", ascending=False) print(\"\\nDisplaying head of sorted dataframe: \") print(evals.head()) In\u00a0[\u00a0]: Copied! <pre>from matplotlib import pyplot as plt\n\n\ndef plot_cdf(values, metric=\"predictive_accuracy\"):\n    max_val = max(values)\n    n, bins, patches = plt.hist(values, density=True, histtype=\"step\", cumulative=True, linewidth=3)\n    patches[0].set_xy(patches[0].get_xy()[:-1])\n    plt.xlim(max(0, min(values) - 0.1), 1)\n    plt.title(\"CDF\")\n    plt.xlabel(metric)\n    plt.ylabel(\"Likelihood\")\n    plt.grid(visible=True, which=\"major\", linestyle=\"-\")\n    plt.minorticks_on()\n    plt.grid(visible=True, which=\"minor\", linestyle=\"--\")\n    plt.axvline(max_val, linestyle=\"--\", color=\"gray\")\n    plt.text(max_val, 0, \"%.3f\" % max_val, fontsize=9)\n    plt.show()\n\n\nplot_cdf(evals.value, metric)\n</pre> from matplotlib import pyplot as plt   def plot_cdf(values, metric=\"predictive_accuracy\"):     max_val = max(values)     n, bins, patches = plt.hist(values, density=True, histtype=\"step\", cumulative=True, linewidth=3)     patches[0].set_xy(patches[0].get_xy()[:-1])     plt.xlim(max(0, min(values) - 0.1), 1)     plt.title(\"CDF\")     plt.xlabel(metric)     plt.ylabel(\"Likelihood\")     plt.grid(visible=True, which=\"major\", linestyle=\"-\")     plt.minorticks_on()     plt.grid(visible=True, which=\"minor\", linestyle=\"--\")     plt.axvline(max_val, linestyle=\"--\", color=\"gray\")     plt.text(max_val, 0, \"%.3f\" % max_val, fontsize=9)     plt.show()   plot_cdf(evals.value, metric) <p>This CDF plot shows that for the given task, based on the results of the runs uploaded, it is almost certain to achieve an accuracy above 52%, i.e., with non-zero probability. While the maximum accuracy seen till now is 96.5%.</p> In\u00a0[\u00a0]: Copied! <pre>import numpy as np\nimport pandas as pd\n\n\ndef plot_flow_compare(evaluations, top_n=10, metric=\"predictive_accuracy\"):\n    # Collecting the top 10 performing unique flow_id\n    flow_ids = evaluations.flow_id.unique()[:top_n]\n\n    df = pd.DataFrame()\n    # Creating a data frame containing only the metric values of the selected flows\n    #   assuming evaluations is sorted in decreasing order of metric\n    for i in range(len(flow_ids)):\n        flow_values = evaluations[evaluations.flow_id == flow_ids[i]].value\n        df = pd.concat([df, flow_values], ignore_index=True, axis=1)\n    fig, axs = plt.subplots()\n    df.boxplot()\n    axs.set_title(\"Boxplot comparing \" + metric + \" for different flows\")\n    axs.set_ylabel(metric)\n    axs.set_xlabel(\"Flow ID\")\n    axs.set_xticklabels(flow_ids)\n    axs.grid(which=\"major\", linestyle=\"-\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")\n    axs.minorticks_on()\n    axs.grid(which=\"minor\", linestyle=\"--\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")\n    # Counting the number of entries for each flow in the data frame\n    #   which gives the number of runs for each flow\n    flow_freq = list(df.count(axis=0, numeric_only=True))\n    for i in range(len(flow_ids)):\n        axs.text(i + 1.05, np.nanmin(df.values), str(flow_freq[i]) + \"\\nrun(s)\", fontsize=7)\n    plt.show()\n\n\nplot_flow_compare(evals, metric=metric, top_n=10)\n</pre> import numpy as np import pandas as pd   def plot_flow_compare(evaluations, top_n=10, metric=\"predictive_accuracy\"):     # Collecting the top 10 performing unique flow_id     flow_ids = evaluations.flow_id.unique()[:top_n]      df = pd.DataFrame()     # Creating a data frame containing only the metric values of the selected flows     #   assuming evaluations is sorted in decreasing order of metric     for i in range(len(flow_ids)):         flow_values = evaluations[evaluations.flow_id == flow_ids[i]].value         df = pd.concat([df, flow_values], ignore_index=True, axis=1)     fig, axs = plt.subplots()     df.boxplot()     axs.set_title(\"Boxplot comparing \" + metric + \" for different flows\")     axs.set_ylabel(metric)     axs.set_xlabel(\"Flow ID\")     axs.set_xticklabels(flow_ids)     axs.grid(which=\"major\", linestyle=\"-\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")     axs.minorticks_on()     axs.grid(which=\"minor\", linestyle=\"--\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")     # Counting the number of entries for each flow in the data frame     #   which gives the number of runs for each flow     flow_freq = list(df.count(axis=0, numeric_only=True))     for i in range(len(flow_ids)):         axs.text(i + 1.05, np.nanmin(df.values), str(flow_freq[i]) + \"\\nrun(s)\", fontsize=7)     plt.show()   plot_flow_compare(evals, metric=metric, top_n=10) <p>The boxplots below show how the flows perform across multiple runs on the chosen task. The green horizontal lines represent the median accuracy of all the runs for that flow (number of runs denoted at the bottom of the boxplots). The higher the green line, the better the flow is for the task at hand. The ordering of the flows are in the descending order of the higest accuracy value seen under that flow.</p> <p>Printing the corresponding flow names for the top 10 performing flow IDs</p> In\u00a0[\u00a0]: Copied! <pre>top_n = 10\nflow_ids = evals.flow_id.unique()[:top_n]\nflow_names = evals.flow_name.unique()[:top_n]\nfor i in range(top_n):\n    print((flow_ids[i], flow_names[i]))\n</pre> top_n = 10 flow_ids = evals.flow_id.unique()[:top_n] flow_names = evals.flow_name.unique()[:top_n] for i in range(top_n):     print((flow_ids[i], flow_names[i])) In\u00a0[\u00a0]: Copied! <pre>evals_setups = openml.evaluations.list_evaluations_setups(\n    function=\"predictive_accuracy\", tasks=[31], size=100, sort_order=\"desc\"\n)\n\nprint(evals_setups.head())\n</pre> evals_setups = openml.evaluations.list_evaluations_setups(     function=\"predictive_accuracy\", tasks=[31], size=100, sort_order=\"desc\" )  print(evals_setups.head()) <p>Return evaluations for flow_id in descending order based on predictive_accuracy with hyperparameters. parameters_in_separate_columns returns parameters in separate columns</p> In\u00a0[\u00a0]: Copied! <pre>evals_setups = openml.evaluations.list_evaluations_setups(\n    function=\"predictive_accuracy\", flows=[6767], size=100, parameters_in_separate_columns=True\n)\n\nprint(evals_setups.head(10))\n\n# License: BSD 3-Clause\n</pre> evals_setups = openml.evaluations.list_evaluations_setups(     function=\"predictive_accuracy\", flows=[6767], size=100, parameters_in_separate_columns=True )  print(evals_setups.head(10))  # License: BSD 3-Clause"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#fetching-evaluations","title":"Fetching Evaluations\u00b6","text":"<p>Evaluations contain a concise summary of the results of all runs made. Each evaluation provides information on the dataset used, the flow applied, the setup used, the metric evaluated, and the result obtained on the metric, for each such run made. These collection of results can be used for efficient benchmarking of an algorithm and also allow transparent reuse of results from previous experiments on similar parameters.</p> <p>In this example, we shall do the following:</p> <ul> <li>Retrieve evaluations based on different metrics</li> <li>Fetch evaluations pertaining to a specific task</li> <li>Sort the obtained results in descending order of the metric</li> <li>Plot a cumulative distribution function for the evaluations</li> <li>Compare the top 10 performing flows based on the evaluation performance</li> <li>Retrieve evaluations with hyperparameter settings</li> </ul>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#listing-evaluations","title":"Listing evaluations\u00b6","text":"<p>Evaluations can be retrieved from the database in the chosen output format. Required filters can be applied to retrieve results from runs as required.</p> <p>We shall retrieve a small set (only 10 entries) to test the listing function for evaluations</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#viewing-a-sample-task","title":"Viewing a sample task\u00b6","text":"<p>Over here we shall briefly take a look at the details of the task. We will start by displaying a simple supervised classification task:</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#obtaining-all-the-evaluations-for-the-task","title":"Obtaining all the evaluations for the task\u00b6","text":"<p>We'll now obtain all the evaluations that were uploaded for the task we displayed previously. Note that we now filter the evaluations based on another parameter 'task'.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#obtaining-cdf-of-metric-for-chosen-task","title":"Obtaining CDF of metric for chosen task\u00b6","text":"<p>We shall now analyse how the performance of various flows have been on this task, by seeing the likelihood of the accuracy obtained across all runs. We shall now plot a cumulative distributive function (CDF) for the accuracies obtained.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#comparing-top-10-performing-flows","title":"Comparing top 10 performing flows\u00b6","text":"<p>Let us now try to see which flows generally performed the best for this task. For this, we shall compare the top performing flows.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#obtaining-evaluations-with-hyperparameter-settings","title":"Obtaining evaluations with hyperparameter settings\u00b6","text":"<p>We'll now obtain the evaluations of a task and a flow with the hyperparameters</p> <p>List evaluations in descending order based on predictive_accuracy with hyperparameters</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/","title":"Preparing tasks and scikit-learn models","text":"<p>Measuring runtimes for Scikit-learn models</p> <p>The runtime of machine learning models on specific datasets can be a deciding factor on the choice of algorithms, especially for benchmarking and comparison purposes. OpenML's scikit-learn extension provides runtime data from runs of model fit and prediction on tasks or datasets, for both the CPU-clock as well as the actual wallclock-time incurred. The objective of this example is to illustrate how to retrieve such timing measures, and also offer some potential means of usage and interpretation of the same.</p> <p>It should be noted that there are multiple levels at which parallelism can occur.</p> <ul> <li><p>At the outermost level, OpenML tasks contain fixed data splits, on which the defined model/flow is executed. Thus, a model can be fit on each OpenML dataset fold in parallel using the <code>n_jobs</code> parameter to <code>run_model_on_task</code> or <code>run_flow_on_task</code> (illustrated under Case 2 &amp; 3 below).</p> </li> <li><p>The model/flow specified can also include scikit-learn models that perform their own parallelization. For instance, by specifying <code>n_jobs</code> in a Random Forest model definition (covered under Case 2 below).</p> </li> <li><p>The sklearn model can further be an HPO estimator and contain it's own parallelization. If the base estimator used also supports <code>parallelization</code>, then there's at least a 2-level nested definition for parallelization possible (covered under Case 3 below).</p> </li> </ul> <p>We shall cover these 5 representative scenarios for:</p> <ul> <li><p>(Case 1) Retrieving runtimes for Random Forest training and prediction on each of the cross-validation folds</p> </li> <li><p>(Case 2) Testing the above setting in a parallel setup and monitor the difference using runtimes retrieved</p> </li> <li><p>(Case 3) Comparing RandomSearchCV and GridSearchCV on the above task based on runtimes</p> </li> <li><p>(Case 4) Running models that don't run in parallel or models which scikit-learn doesn't parallelize</p> </li> <li><p>(Case 5) Running models that do not release the Python Global Interpreter Lock (GIL)</p> </li> </ul> <p>import openml import numpy as np from matplotlib import pyplot as plt from joblib.parallel import parallel_backend</p> <p>from sklearn.naive_bayes import GaussianNB from sklearn.tree import DecisionTreeClassifier from sklearn.neural_network import MLPClassifier from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV, RandomizedSearchCV</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 167119\n\ntask = openml.tasks.get_task(task_id)\nprint(task)\n\n# Viewing associated data\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n\n\n# Creating utility function\ndef print_compare_runtimes(measures):\n    for repeat, val1 in measures[\"usercpu_time_millis_training\"].items():\n        for fold, val2 in val1.items():\n            print(\n                \"Repeat #{}-Fold #{}: CPU-{:.3f} vs Wall-{:.3f}\".format(\n                    repeat, fold, val2, measures[\"wall_clock_time_millis_training\"][repeat][fold]\n                )\n            )\n</pre> task_id = 167119  task = openml.tasks.get_task(task_id) print(task)  # Viewing associated data n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) )   # Creating utility function def print_compare_runtimes(measures):     for repeat, val1 in measures[\"usercpu_time_millis_training\"].items():         for fold, val2 in val1.items():             print(                 \"Repeat #{}-Fold #{}: CPU-{:.3f} vs Wall-{:.3f}\".format(                     repeat, fold, val2, measures[\"wall_clock_time_millis_training\"][repeat][fold]                 )             ) In\u00a0[\u00a0]: Copied! <pre>clf = RandomForestClassifier(n_estimators=10)\n\nrun1 = openml.runs.run_model_on_task(\n    model=clf,\n    task=task,\n    upload_flow=False,\n    avoid_duplicate_runs=False,\n)\nmeasures = run1.fold_evaluations\n\nprint(\"The timing and performance metrics available: \")\nfor key in measures.keys():\n    print(key)\nprint()\n\nprint(\n    \"The performance metric is recorded under `predictive_accuracy` per \"\n    \"fold and can be retrieved as: \"\n)\nfor repeat, val1 in measures[\"predictive_accuracy\"].items():\n    for fold, val2 in val1.items():\n        print(\"Repeat #{}-Fold #{}: {:.4f}\".format(repeat, fold, val2))\n    print()\n</pre> clf = RandomForestClassifier(n_estimators=10)  run1 = openml.runs.run_model_on_task(     model=clf,     task=task,     upload_flow=False,     avoid_duplicate_runs=False, ) measures = run1.fold_evaluations  print(\"The timing and performance metrics available: \") for key in measures.keys():     print(key) print()  print(     \"The performance metric is recorded under `predictive_accuracy` per \"     \"fold and can be retrieved as: \" ) for repeat, val1 in measures[\"predictive_accuracy\"].items():     for fold, val2 in val1.items():         print(\"Repeat #{}-Fold #{}: {:.4f}\".format(repeat, fold, val2))     print() <p>The remaining entries recorded in <code>measures</code> are the runtime records related as:</p> <p>usercpu_time_millis = usercpu_time_millis_training + usercpu_time_millis_testing</p> <p>wall_clock_time_millis = wall_clock_time_millis_training + wall_clock_time_millis_testing</p> <p>The timing measures recorded as <code>*_millis_training</code> contain the per repeat-per fold timing incurred for the execution of the <code>.fit()</code> procedure of the model. For <code>usercpu_time_*</code> the time recorded using <code>time.process_time()</code> is converted to <code>milliseconds</code> and stored. Similarly, <code>time.time()</code> is used to record the time entry for <code>wall_clock_time_*</code>. The <code>*_millis_testing</code> entry follows the same procedure but for time taken for the <code>.predict()</code> procedure.</p> <p>Comparing the CPU and wall-clock training times of the Random Forest model</p> In\u00a0[\u00a0]: Copied! <pre>print_compare_runtimes(measures)\n</pre> print_compare_runtimes(measures) In\u00a0[\u00a0]: Copied! <pre>clf = RandomForestClassifier(n_estimators=10, n_jobs=2)\n\nrun2 = openml.runs.run_model_on_task(\n    model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n)\nmeasures = run2.fold_evaluations\n# The wall-clock time recorded per fold should be lesser than Case 1 above\nprint_compare_runtimes(measures)\n</pre> clf = RandomForestClassifier(n_estimators=10, n_jobs=2)  run2 = openml.runs.run_model_on_task(     model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False ) measures = run2.fold_evaluations # The wall-clock time recorded per fold should be lesser than Case 1 above print_compare_runtimes(measures) <p>Running a Random Forest model on an OpenML task in parallel (all cores available):</p> In\u00a0[\u00a0]: Copied! <pre># Redefining the model to use all available cores with `n_jobs=-1`\nclf = RandomForestClassifier(n_estimators=10, n_jobs=-1)\n\nrun3 = openml.runs.run_model_on_task(\n    model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n)\nmeasures = run3.fold_evaluations\n</pre> # Redefining the model to use all available cores with `n_jobs=-1` clf = RandomForestClassifier(n_estimators=10, n_jobs=-1)  run3 = openml.runs.run_model_on_task(     model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False ) measures = run3.fold_evaluations <p>The wall-clock time recorded per fold should be lesser than the case above, if more than 2 CPU cores are available. The speed-up is more pronounced for larger datasets. print_compare_runtimes(measures)</p> <p>We can now observe that the ratio of CPU time to wallclock time is lower than in case 1. This happens because joblib by default spawns subprocesses for the workloads for which CPU time cannot be tracked. Therefore, interpreting the reported CPU and wallclock time requires knowledge of the parallelization applied at runtime.</p> <p>Running the same task with a different parallel backend. Joblib provides multiple backends: {<code>loky</code> (default), <code>multiprocessing</code>, <code>dask</code>, <code>threading</code>, <code>sequential</code>}. The backend can be explicitly set using a joblib context manager. The behaviour of the job distribution can change and therefore the scale of runtimes recorded too.</p> In\u00a0[\u00a0]: Copied! <pre>with parallel_backend(backend=\"multiprocessing\", n_jobs=-1):\n    run3_ = openml.runs.run_model_on_task(\n        model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n    )\nmeasures = run3_.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> with parallel_backend(backend=\"multiprocessing\", n_jobs=-1):     run3_ = openml.runs.run_model_on_task(         model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False     ) measures = run3_.fold_evaluations print_compare_runtimes(measures) <p>The CPU time interpretation becomes ambiguous when jobs are distributed over an unknown number of cores or when subprocesses are spawned for which the CPU time cannot be tracked, as in the examples above. It is impossible for OpenML-Python to capture the availability of the number of cores/threads, their eventual utilisation and whether workloads are executed in subprocesses, for various cases that can arise as demonstrated in the rest of the example. Therefore, the final interpretation of the runtimes is left to the <code>user</code>.</p> In\u00a0[\u00a0]: Copied! <pre>from sklearn.model_selection import GridSearchCV\n\nclf = RandomForestClassifier(n_estimators=10, n_jobs=2)\n\n# GridSearchCV model\nn_iter = 5\ngrid_pipe = GridSearchCV(\n    estimator=clf,\n    param_grid={\"n_estimators\": np.linspace(start=1, stop=50, num=n_iter).astype(int).tolist()},\n    cv=2,\n    n_jobs=2,\n)\n\nrun4 = openml.runs.run_model_on_task(\n    model=grid_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2\n)\nmeasures = run4.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> from sklearn.model_selection import GridSearchCV  clf = RandomForestClassifier(n_estimators=10, n_jobs=2)  # GridSearchCV model n_iter = 5 grid_pipe = GridSearchCV(     estimator=clf,     param_grid={\"n_estimators\": np.linspace(start=1, stop=50, num=n_iter).astype(int).tolist()},     cv=2,     n_jobs=2, )  run4 = openml.runs.run_model_on_task(     model=grid_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2 ) measures = run4.fold_evaluations print_compare_runtimes(measures) <p>Like any optimisation problem, scikit-learn's HPO estimators also generate a sequence of configurations which are evaluated, using which the best found configuration is tracked throughout the trace. The OpenML run object stores these traces as OpenMLRunTrace objects accessible using keys of the pattern (repeat, fold, iterations). Here <code>fold</code> implies the outer-cross validation fold as obtained from the task data splits in OpenML. GridSearchCV here performs grid search over the inner-cross validation folds as parameterized by the <code>cv</code> parameter. Since <code>GridSearchCV</code> in this example performs a <code>2-fold</code> cross validation, the runtime recorded per repeat-per fold in the run object is for the entire <code>fit()</code> procedure of GridSearchCV thus subsuming the runtimes of the 2-fold (inner) CV search performed.</p> In\u00a0[\u00a0]: Copied! <pre># We earlier extracted the number of repeats and folds for this task:\nprint(\"# repeats: {}\\n# folds: {}\".format(n_repeats, n_folds))\n\n# To extract the training runtime of the first repeat, first fold:\nprint(run4.fold_evaluations[\"wall_clock_time_millis_training\"][0][0])\n</pre> # We earlier extracted the number of repeats and folds for this task: print(\"# repeats: {}\\n# folds: {}\".format(n_repeats, n_folds))  # To extract the training runtime of the first repeat, first fold: print(run4.fold_evaluations[\"wall_clock_time_millis_training\"][0][0]) <p>To extract the training runtime of the 1-st repeat, 4-th (outer) fold and also to fetch the parameters and performance of the evaluations made during the 1-st repeat, 4-th fold evaluation by the Grid Search model.</p> In\u00a0[\u00a0]: Copied! <pre>_repeat = 0\n_fold = 3\nprint(\n    \"Total runtime for repeat {}'s fold {}: {:4f} ms\".format(\n        _repeat, _fold, run4.fold_evaluations[\"wall_clock_time_millis_training\"][_repeat][_fold]\n    )\n)\nfor i in range(n_iter):\n    key = (_repeat, _fold, i)\n    r = run4.trace.trace_iterations[key]\n    print(\n        \"n_estimators: {:&gt;2} - score: {:.3f}\".format(\n            r.parameters[\"parameter_n_estimators\"], r.evaluation\n        )\n    )\n</pre> _repeat = 0 _fold = 3 print(     \"Total runtime for repeat {}'s fold {}: {:4f} ms\".format(         _repeat, _fold, run4.fold_evaluations[\"wall_clock_time_millis_training\"][_repeat][_fold]     ) ) for i in range(n_iter):     key = (_repeat, _fold, i)     r = run4.trace.trace_iterations[key]     print(         \"n_estimators: {:&gt;2} - score: {:.3f}\".format(             r.parameters[\"parameter_n_estimators\"], r.evaluation         )     ) <p>Scikit-learn's HPO estimators also come with an argument <code>refit=True</code> as a default. In our previous model definition it was set to True by default, which meant that the best found hyperparameter configuration was used to refit or retrain the model without any inner cross validation. This extra refit time measure is provided by the scikit-learn model as the attribute <code>refit_time_</code>. This time is included in the <code>wall_clock_time_millis_training</code> measure.</p> <p>For non-HPO estimators, <code>wall_clock_time_millis = wall_clock_time_millis_training + wall_clock_time_millis_testing</code>.</p> <p>For HPO estimators, <code>wall_clock_time_millis = wall_clock_time_millis_training + wall_clock_time_millis_testing + refit_time</code>.</p> <p>This refit time can therefore be explicitly extracted in this manner:</p> In\u00a0[\u00a0]: Copied! <pre>def extract_refit_time(run, repeat, fold):\n    refit_time = (\n        run.fold_evaluations[\"wall_clock_time_millis\"][repeat][fold]\n        - run.fold_evaluations[\"wall_clock_time_millis_training\"][repeat][fold]\n        - run.fold_evaluations[\"wall_clock_time_millis_testing\"][repeat][fold]\n    )\n    return refit_time\n\n\nfor repeat in range(n_repeats):\n    for fold in range(n_folds):\n        print(\n            \"Repeat #{}-Fold #{}: {:.4f}\".format(\n                repeat, fold, extract_refit_time(run4, repeat, fold)\n            )\n        )\n</pre>  def extract_refit_time(run, repeat, fold):     refit_time = (         run.fold_evaluations[\"wall_clock_time_millis\"][repeat][fold]         - run.fold_evaluations[\"wall_clock_time_millis_training\"][repeat][fold]         - run.fold_evaluations[\"wall_clock_time_millis_testing\"][repeat][fold]     )     return refit_time   for repeat in range(n_repeats):     for fold in range(n_folds):         print(             \"Repeat #{}-Fold #{}: {:.4f}\".format(                 repeat, fold, extract_refit_time(run4, repeat, fold)             )         ) <p>Along with the GridSearchCV already used above, we demonstrate how such optimisation traces can be retrieved by showing an application of these traces - comparing the speed of finding the best configuration using RandomizedSearchCV and GridSearchCV available with scikit-learn.</p> In\u00a0[\u00a0]: Copied! <pre># RandomizedSearchCV model\nrs_pipe = RandomizedSearchCV(\n    estimator=clf,\n    param_distributions={\n        \"n_estimators\": np.linspace(start=1, stop=50, num=15).astype(int).tolist()\n    },\n    cv=2,\n    n_iter=n_iter,\n    n_jobs=2,\n)\nrun5 = openml.runs.run_model_on_task(\n    model=rs_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2\n)\n</pre> # RandomizedSearchCV model rs_pipe = RandomizedSearchCV(     estimator=clf,     param_distributions={         \"n_estimators\": np.linspace(start=1, stop=50, num=15).astype(int).tolist()     },     cv=2,     n_iter=n_iter,     n_jobs=2, ) run5 = openml.runs.run_model_on_task(     model=rs_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2 ) <p>Since for the call to <code>openml.runs.run_model_on_task</code> the parameter <code>n_jobs</code> is set to its default <code>None</code>, the evaluations across the OpenML folds are not parallelized. Hence, the time recorded is agnostic to the <code>n_jobs</code> being set at both the HPO estimator <code>GridSearchCV</code> as well as the base estimator <code>RandomForestClassifier</code> in this case. The OpenML extension only records the time taken for the completion of the complete <code>fit()</code> call, per-repeat per-fold.</p> <p>This notion can be used to extract and plot the best found performance per fold by the HPO model and the corresponding time taken for search across that fold. Moreover, since <code>n_jobs=None</code> for <code>openml.runs.run_model_on_task</code> the runtimes per fold can be cumulatively added to plot the trace against time.</p> In\u00a0[\u00a0]: Copied! <pre>def extract_trace_data(run, n_repeats, n_folds, n_iter, key=None):\n    key = \"wall_clock_time_millis_training\" if key is None else key\n    data = {\"score\": [], \"runtime\": []}\n    for i_r in range(n_repeats):\n        for i_f in range(n_folds):\n            data[\"runtime\"].append(run.fold_evaluations[key][i_r][i_f])\n            for i_i in range(n_iter):\n                r = run.trace.trace_iterations[(i_r, i_f, i_i)]\n                if r.selected:\n                    data[\"score\"].append(r.evaluation)\n                    break\n    return data\n\n\ndef get_incumbent_trace(trace):\n    best_score = 1\n    inc_trace = []\n    for i, r in enumerate(trace):\n        if i == 0 or (1 - r) &lt; best_score:\n            best_score = 1 - r\n        inc_trace.append(best_score)\n    return inc_trace\n\n\ngrid_data = extract_trace_data(run4, n_repeats, n_folds, n_iter)\nrs_data = extract_trace_data(run5, n_repeats, n_folds, n_iter)\n\nplt.clf()\nplt.plot(\n    np.cumsum(grid_data[\"runtime\"]), get_incumbent_trace(grid_data[\"score\"]), label=\"Grid Search\"\n)\nplt.plot(\n    np.cumsum(rs_data[\"runtime\"]), get_incumbent_trace(rs_data[\"score\"]), label=\"Random Search\"\n)\nplt.xscale(\"log\")\nplt.yscale(\"log\")\nplt.xlabel(\"Wallclock time (in milliseconds)\")\nplt.ylabel(\"1 - Accuracy\")\nplt.title(\"Optimisation Trace Comparison\")\nplt.legend()\nplt.show()\n</pre> def extract_trace_data(run, n_repeats, n_folds, n_iter, key=None):     key = \"wall_clock_time_millis_training\" if key is None else key     data = {\"score\": [], \"runtime\": []}     for i_r in range(n_repeats):         for i_f in range(n_folds):             data[\"runtime\"].append(run.fold_evaluations[key][i_r][i_f])             for i_i in range(n_iter):                 r = run.trace.trace_iterations[(i_r, i_f, i_i)]                 if r.selected:                     data[\"score\"].append(r.evaluation)                     break     return data   def get_incumbent_trace(trace):     best_score = 1     inc_trace = []     for i, r in enumerate(trace):         if i == 0 or (1 - r) &lt; best_score:             best_score = 1 - r         inc_trace.append(best_score)     return inc_trace   grid_data = extract_trace_data(run4, n_repeats, n_folds, n_iter) rs_data = extract_trace_data(run5, n_repeats, n_folds, n_iter)  plt.clf() plt.plot(     np.cumsum(grid_data[\"runtime\"]), get_incumbent_trace(grid_data[\"score\"]), label=\"Grid Search\" ) plt.plot(     np.cumsum(rs_data[\"runtime\"]), get_incumbent_trace(rs_data[\"score\"]), label=\"Random Search\" ) plt.xscale(\"log\") plt.yscale(\"log\") plt.xlabel(\"Wallclock time (in milliseconds)\") plt.ylabel(\"1 - Accuracy\") plt.title(\"Optimisation Trace Comparison\") plt.legend() plt.show() In\u00a0[\u00a0]: Copied! <pre>dt = DecisionTreeClassifier()\n\nrun6 = openml.runs.run_model_on_task(\n    model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2\n)\nmeasures = run6.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> dt = DecisionTreeClassifier()  run6 = openml.runs.run_model_on_task(     model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2 ) measures = run6.fold_evaluations print_compare_runtimes(measures) <p>Although the decision tree does not run in parallel, it can release the <code>Python GIL &lt;https://docs.python.org/dev/glossary.html#term-global-interpreter-lock&gt;</code>_. This can result in surprising runtime measures as demonstrated below:</p> In\u00a0[\u00a0]: Copied! <pre>with parallel_backend(\"threading\", n_jobs=-1):\n    run7 = openml.runs.run_model_on_task(\n        model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False\n    )\nmeasures = run7.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> with parallel_backend(\"threading\", n_jobs=-1):     run7 = openml.runs.run_model_on_task(         model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False     ) measures = run7.fold_evaluations print_compare_runtimes(measures) <p>Running a Neural Network from scikit-learn that uses scikit-learn independent parallelism using libraries such as MKL, OpenBLAS or BLIS.</p> In\u00a0[\u00a0]: Copied! <pre>mlp = MLPClassifier(max_iter=10)\n\nrun8 = openml.runs.run_model_on_task(\n    model=mlp, task=task, upload_flow=False, avoid_duplicate_runs=False\n)\nmeasures = run8.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> mlp = MLPClassifier(max_iter=10)  run8 = openml.runs.run_model_on_task(     model=mlp, task=task, upload_flow=False, avoid_duplicate_runs=False ) measures = run8.fold_evaluations print_compare_runtimes(measures) In\u00a0[\u00a0]: Copied! <pre>clf = GaussianNB()\n\nwith parallel_backend(\"multiprocessing\", n_jobs=-1):\n    run9 = openml.runs.run_model_on_task(\n        model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n    )\nmeasures = run9.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> clf = GaussianNB()  with parallel_backend(\"multiprocessing\", n_jobs=-1):     run9 = openml.runs.run_model_on_task(         model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False     ) measures = run9.fold_evaluations print_compare_runtimes(measures)"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#preparing-tasks-and-scikit-learn-models","title":"Preparing tasks and scikit-learn models\u00b6","text":""},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-1-running-a-random-forest-model-on-an-openml-task","title":"Case 1: Running a Random Forest model on an OpenML task\u00b6","text":"<p>We'll run a Random Forest model and obtain an OpenML run object. We can see the evaluations recorded per fold for the dataset and the information available for this run.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-2-running-scikit-learn-model-on-an-openml-task-in-parallel","title":"Case 2: Running Scikit-learn model on an OpenML task in parallel\u00b6","text":"<p>Redefining the model to allow parallelism with <code>n_jobs=2</code> (2 cores)</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes","title":"Case 3: Running and benchmarking HPO algorithms with their runtimes\u00b6","text":"<p>We shall now optimize a similar RandomForest model for the same task using scikit-learn's HPO support by using GridSearchCV to optimize our earlier RandomForest model's hyperparameter <code>n_estimators</code>. Scikit-learn also provides a <code>refit_time_</code> for such HPO models, i.e., the time incurred by training and evaluating the model on the best found parameter setting. This is included in the <code>wall_clock_time_millis_training</code> measure recorded.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-4-running-models-that-scikit-learn-doesnt-parallelize","title":"Case 4: Running models that scikit-learn doesn't parallelize\u00b6","text":"<p>Both scikit-learn and OpenML depend on parallelism implemented through <code>joblib</code>. However, there can be cases where either models cannot be parallelized or don't depend on joblib for its parallelism. 2 such cases are illustrated below.</p> <p>Running a Decision Tree model that doesn't support parallelism implicitly, but using OpenML to parallelize evaluations for the outer-cross validation folds.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-5-running-scikit-learn-models-that-dont-release-gil","title":"Case 5: Running Scikit-learn models that don't release GIL\u00b6","text":"<p>Certain Scikit-learn models do not release the Python GIL and are also not executed in parallel via a BLAS library. In such cases, the CPU times and wallclock times are most likely trustworthy. Note however that only very few models such as naive Bayes models are of this kind.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#summmary","title":"Summmary\u00b6","text":"<p>The scikit-learn extension for OpenML-Python records model runtimes for the CPU-clock and the wall-clock times. The above examples illustrated how these recorded runtimes can be extracted when using a scikit-learn model and under parallel setups too. To summarize, the scikit-learn extension measures the:</p> <ul> <li><p><code>CPU-time</code> &amp; <code>wallclock-time</code> for the whole run</p> <ul> <li>A run here corresponds to a call to <code>run_model_on_task</code> or <code>run_flow_on_task</code></li> <li>The recorded time is for the model fit for each of the outer-cross validations folds, i.e., the OpenML data splits</li> </ul> </li> <li><p>Python's <code>time</code> module is used to compute the runtimes</p> <ul> <li><code>CPU-time</code> is recorded using the responses of <code>time.process_time()</code></li> <li><code>wallclock-time</code> is recorded using the responses of <code>time.time()</code></li> </ul> </li> <li><p>The timings recorded by OpenML per outer-cross validation fold is agnostic to model parallelisation</p> <ul> <li>The wallclock times reported in Case 2 above highlights the speed-up on using <code>n_jobs=-1</code> in comparison to <code>n_jobs=2</code>, since the timing recorded by OpenML is for the entire <code>fit()</code> procedure, whereas the parallelisation is performed inside <code>fit()</code> by scikit-learn</li> <li>The CPU-time for models that are run in parallel can be difficult to interpret</li> </ul> </li> <li><p><code>CPU-time</code> &amp; <code>wallclock-time</code> for each search per outer fold in an HPO run</p> <ul> <li>Reports the total time for performing search on each of the OpenML data split, subsuming any sort of parallelism that happened as part of the HPO estimator or the underlying base estimator</li> <li>Also allows extraction of the <code>refit_time</code> that scikit-learn measures using <code>time.time()</code> for retraining the model per outer fold, for the best found configuration</li> </ul> </li> <li><p><code>CPU-time</code> &amp; <code>wallclock-time</code> for models that scikit-learn doesn't parallelize</p> <ul> <li>Models like Decision Trees or naive Bayes don't parallelize and thus both the wallclock and CPU times are similar in runtime for the OpenML call</li> <li>However, models implemented in Cython, such as the Decision Trees can release the GIL and still run in parallel if a <code>threading</code> backend is used by joblib.</li> <li>Scikit-learn Neural Networks can undergo parallelization implicitly owing to thread-level parallelism involved in the linear algebraic operations and thus the wallclock-time and CPU-time can differ.</li> </ul> </li> </ul> <p>Because of all the cases mentioned above it is crucial to understand which case is triggered when reporting runtimes for scikit-learn models measured with OpenML-Python! License: BSD 3-Clause</p>"},{"location":"examples/30_extended/flow_id_tutorial/","title":"Obtaining Flow IDs","text":"In\u00a0[\u00a0]: Copied! <pre>import sklearn.tree\n\nimport openml\n</pre> import sklearn.tree  import openml <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre># Defining a classifier\nclf = sklearn.tree.DecisionTreeClassifier()\n</pre> # Defining a classifier clf = sklearn.tree.DecisionTreeClassifier() In\u00a0[\u00a0]: Copied! <pre>flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish()\nflow_id = flow.flow_id\nprint(flow_id)\n</pre> flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish() flow_id = flow.flow_id print(flow_id) <p>This piece of code is rather involved. First, it retrieves a :class:<code>~openml.extensions.Extension</code> which is registered and can handle the given model, in our case it is :class:<code>openml.extensions.sklearn.SklearnExtension</code>. Second, the extension converts the classifier into an instance of :class:<code>openml.OpenMLFlow</code>. Third and finally, the publish method checks whether the current flow is already present on OpenML. If not, it uploads the flow, otherwise, it updates the current instance with all information computed by the server (which is obviously also done when uploading/publishing a flow).</p> <p>To simplify the usage we have created a helper function which automates all these steps:</p> In\u00a0[\u00a0]: Copied! <pre>flow_id = openml.flows.get_flow_id(model=clf)\nprint(flow_id)\n</pre> flow_id = openml.flows.get_flow_id(model=clf) print(flow_id) In\u00a0[\u00a0]: Copied! <pre>print(flow.name, flow.external_version)\n</pre> print(flow.name, flow.external_version) <p>The name and external version are automatically added to a flow when constructing it from a model. We can then use them to retrieve the flow id as follows:</p> In\u00a0[\u00a0]: Copied! <pre>flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version)\nprint(flow_id)\n</pre> flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version) print(flow_id) <p>We can also retrieve all flows for a given name:</p> In\u00a0[\u00a0]: Copied! <pre>flow_ids = openml.flows.get_flow_id(name=flow.name)\nprint(flow_ids)\n</pre> flow_ids = openml.flows.get_flow_id(name=flow.name) print(flow_ids) <p>This also works with the actual model (generalizing the first part of this example):</p> In\u00a0[\u00a0]: Copied! <pre>flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False)\nprint(flow_ids)\n</pre> flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False) print(flow_ids) In\u00a0[\u00a0]: Copied! <pre># Deactivating test configuration\nopenml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> # Deactivating test configuration openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/flow_id_tutorial/#obtaining-flow-ids","title":"Obtaining Flow IDs\u00b6","text":"<p>This tutorial discusses different ways to obtain the ID of a flow in order to perform further analysis.</p>"},{"location":"examples/30_extended/flow_id_tutorial/#1-obtaining-a-flow-given-a-classifier","title":"1. Obtaining a flow given a classifier\u00b6","text":""},{"location":"examples/30_extended/flow_id_tutorial/#2-obtaining-a-flow-given-its-name","title":"2. Obtaining a flow given its name\u00b6","text":"<p>The schema of a flow is given in XSD ( here).  # noqa E501 Only two fields are required, a unique name, and an external version. While it should be pretty obvious why we need a name, the need for the additional external version information might not be immediately clear. However, this information is very important as it allows to have multiple flows with the same name for different versions of a software. This might be necessary if an algorithm or implementation introduces, renames or drop hyperparameters over time.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/","title":"Flows and Runs","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree\n</pre> import openml from sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre># NOTE: We are using dataset 68 from the test server: https://test.openml.org/d/68\ndataset = openml.datasets.get_dataset(68)\nX, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nclf = neighbors.KNeighborsClassifier(n_neighbors=1)\nclf.fit(X, y)\n</pre> # NOTE: We are using dataset 68 from the test server: https://test.openml.org/d/68 dataset = openml.datasets.get_dataset(68) X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) clf = neighbors.KNeighborsClassifier(n_neighbors=1) clf.fit(X, y) <p>You can also ask for meta-data to automatically preprocess the data.</p> <ul> <li>e.g. categorical features -&gt; do feature encoding</li> </ul> In\u00a0[\u00a0]: Copied! <pre>dataset = openml.datasets.get_dataset(17)\nX, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nprint(f\"Categorical features: {categorical_indicator}\")\ntransformer = compose.ColumnTransformer(\n    [(\"one_hot_encoder\", preprocessing.OneHotEncoder(categories=\"auto\"), categorical_indicator)]\n)\nX = transformer.fit_transform(X)\nclf.fit(X, y)\n</pre> dataset = openml.datasets.get_dataset(17) X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) print(f\"Categorical features: {categorical_indicator}\") transformer = compose.ColumnTransformer(     [(\"one_hot_encoder\", preprocessing.OneHotEncoder(categories=\"auto\"), categorical_indicator)] ) X = transformer.fit_transform(X) clf.fit(X, y) In\u00a0[\u00a0]: Copied! <pre># Get a task\ntask = openml.tasks.get_task(403)\n\n# Build any classifier or pipeline\nclf = tree.DecisionTreeClassifier()\n\n# Run the flow\nrun = openml.runs.run_model_on_task(clf, task)\n\nprint(run)\n</pre> # Get a task task = openml.tasks.get_task(403)  # Build any classifier or pipeline clf = tree.DecisionTreeClassifier()  # Run the flow run = openml.runs.run_model_on_task(clf, task)  print(run) <p>Share the run on the OpenML server</p> <p>So far the run is only available locally. By calling the publish function, the run is sent to the OpenML server:</p> In\u00a0[\u00a0]: Copied! <pre>myrun = run.publish()\n# For this tutorial, our configuration publishes to the test server\n# as to not pollute the main server.\nprint(f\"Uploaded to {myrun.openml_url}\")\n</pre> myrun = run.publish() # For this tutorial, our configuration publishes to the test server # as to not pollute the main server. print(f\"Uploaded to {myrun.openml_url}\") <p>We can now also inspect the flow object which was automatically created:</p> In\u00a0[\u00a0]: Copied! <pre>flow = openml.flows.get_flow(run.flow_id)\nprint(flow)\n</pre> flow = openml.flows.get_flow(run.flow_id) print(flow) In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(96)\n\n# OpenML helper functions for sklearn can be plugged in directly for complicated pipelines\nfrom openml.extensions.sklearn import cat, cont\n\npipe = pipeline.Pipeline(\n    steps=[\n        (\n            \"Preprocessing\",\n            compose.ColumnTransformer(\n                [\n                    (\n                        \"categorical\",\n                        preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),\n                        cat,  # returns the categorical feature indices\n                    ),\n                    (\n                        \"continuous\",\n                        impute.SimpleImputer(strategy=\"median\"),\n                        cont,\n                    ),  # returns the numeric feature indices\n                ]\n            ),\n        ),\n        (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),\n    ]\n)\n\nrun = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)\nmyrun = run.publish()\nprint(f\"Uploaded to {myrun.openml_url}\")\n</pre> task = openml.tasks.get_task(96)  # OpenML helper functions for sklearn can be plugged in directly for complicated pipelines from openml.extensions.sklearn import cat, cont  pipe = pipeline.Pipeline(     steps=[         (             \"Preprocessing\",             compose.ColumnTransformer(                 [                     (                         \"categorical\",                         preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),                         cat,  # returns the categorical feature indices                     ),                     (                         \"continuous\",                         impute.SimpleImputer(strategy=\"median\"),                         cont,                     ),  # returns the numeric feature indices                 ]             ),         ),         (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),     ] )  run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False) myrun = run.publish() print(f\"Uploaded to {myrun.openml_url}\") <p>The above pipeline works with the helper functions that internally deal with pandas DataFrame. In the case, pandas is not available, or a NumPy based data processing is the requirement, the above pipeline is presented below to work with NumPy.</p> In\u00a0[\u00a0]: Copied! <pre># Extracting the indices of the categorical columns\nfeatures = task.get_dataset().features\ncategorical_feature_indices = []\nnumeric_feature_indices = []\nfor i in range(len(features)):\n    if features[i].name == task.target_name:\n        continue\n    if features[i].data_type == \"nominal\":\n        categorical_feature_indices.append(i)\n    else:\n        numeric_feature_indices.append(i)\n\npipe = pipeline.Pipeline(\n    steps=[\n        (\n            \"Preprocessing\",\n            compose.ColumnTransformer(\n                [\n                    (\n                        \"categorical\",\n                        preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),\n                        categorical_feature_indices,\n                    ),\n                    (\n                        \"continuous\",\n                        impute.SimpleImputer(strategy=\"median\"),\n                        numeric_feature_indices,\n                    ),\n                ]\n            ),\n        ),\n        (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),\n    ]\n)\n\nrun = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)\nmyrun = run.publish()\nprint(f\"Uploaded to {myrun.openml_url}\")\n</pre> # Extracting the indices of the categorical columns features = task.get_dataset().features categorical_feature_indices = [] numeric_feature_indices = [] for i in range(len(features)):     if features[i].name == task.target_name:         continue     if features[i].data_type == \"nominal\":         categorical_feature_indices.append(i)     else:         numeric_feature_indices.append(i)  pipe = pipeline.Pipeline(     steps=[         (             \"Preprocessing\",             compose.ColumnTransformer(                 [                     (                         \"categorical\",                         preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),                         categorical_feature_indices,                     ),                     (                         \"continuous\",                         impute.SimpleImputer(strategy=\"median\"),                         numeric_feature_indices,                     ),                 ]             ),         ),         (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),     ] )  run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False) myrun = run.publish() print(f\"Uploaded to {myrun.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(96)\n\n# The following lines can then be executed offline:\nrun = openml.runs.run_model_on_task(\n    pipe,\n    task,\n    avoid_duplicate_runs=False,\n    upload_flow=False,\n)\n\n# The run may be stored offline, and the flow will be stored along with it:\nrun.to_filesystem(directory=\"myrun\")\n\n# They may be loaded and uploaded at a later time\nrun = openml.runs.OpenMLRun.from_filesystem(directory=\"myrun\")\nrun.publish()\n\n# Publishing the run will automatically upload the related flow if\n# it does not yet exist on the server.\n</pre> task = openml.tasks.get_task(96)  # The following lines can then be executed offline: run = openml.runs.run_model_on_task(     pipe,     task,     avoid_duplicate_runs=False,     upload_flow=False, )  # The run may be stored offline, and the flow will be stored along with it: run.to_filesystem(directory=\"myrun\")  # They may be loaded and uploaded at a later time run = openml.runs.OpenMLRun.from_filesystem(directory=\"myrun\") run.publish()  # Publishing the run will automatically upload the related flow if # it does not yet exist on the server. <p>Alternatively, one can also directly run flows.</p> In\u00a0[\u00a0]: Copied! <pre># Get a task\ntask = openml.tasks.get_task(403)\n\n# Build any classifier or pipeline\nclf = tree.ExtraTreeClassifier()\n\n# Obtain the scikit-learn extension interface to convert the classifier\n# into a flow object.\nextension = openml.extensions.get_extension_by_model(clf)\nflow = extension.model_to_flow(clf)\n\nrun = openml.runs.run_flow_on_task(flow, task)\n</pre> # Get a task task = openml.tasks.get_task(403)  # Build any classifier or pipeline clf = tree.ExtraTreeClassifier()  # Obtain the scikit-learn extension interface to convert the classifier # into a flow object. extension = openml.extensions.get_extension_by_model(clf) flow = extension.model_to_flow(clf)  run = openml.runs.run_flow_on_task(flow, task) In\u00a0[\u00a0]: Copied! <pre># Easy benchmarking:\nfor task_id in [115]:  # Add further tasks. Disclaimer: they might take some time\n    task = openml.tasks.get_task(task_id)\n    data = openml.datasets.get_dataset(task.dataset_id)\n    clf = neighbors.KNeighborsClassifier(n_neighbors=5)\n\n    run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n    myrun = run.publish()\n    print(f\"kNN on {data.name}: {myrun.openml_url}\")\n</pre> # Easy benchmarking: for task_id in [115]:  # Add further tasks. Disclaimer: they might take some time     task = openml.tasks.get_task(task_id)     data = openml.datasets.get_dataset(task.dataset_id)     clf = neighbors.KNeighborsClassifier(n_neighbors=5)      run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)     myrun = run.publish()     print(f\"kNN on {data.name}: {myrun.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/flows_and_runs_tutorial/#flows-and-runs","title":"Flows and Runs\u00b6","text":"<p>This tutorial covers how to train/run a model and how to upload the results.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#train-machine-learning-models","title":"Train machine learning models\u00b6","text":"<p>Train a scikit-learn model on the data manually.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#runs-easily-explore-models","title":"Runs: Easily explore models\u00b6","text":"<p>We can run (many) scikit-learn algorithms on (many) OpenML tasks.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#it-also-works-with-pipelines","title":"It also works with pipelines\u00b6","text":"<p>When you need to handle 'dirty' data, build pipelines to model then automatically. To demonstrate this using the dataset <code>credit-a &lt;https://test.openml.org/d/16&gt;</code>_ via <code>task &lt;https://test.openml.org/t/96&gt;</code>_ as it contains both numerical and categorical variables and missing values in both.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#running-flows-on-tasks-offline-for-later-upload","title":"Running flows on tasks offline for later upload\u00b6","text":"<p>For those scenarios where there is no access to internet, it is possible to run a model on a task without uploading results or flows to the server immediately.</p> <p>To perform the following line offline, it is required to have been called before such that the task is cached on the local openml cache directory:</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#challenge","title":"Challenge\u00b6","text":"<p>Try to build the best possible models on several OpenML tasks, compare your results with the rest of the class and learn from them. Some tasks you could try (or browse openml.org):</p> <ul> <li>EEG eye state: data_id:<code>1471 &lt;https://www.openml.org/d/1471&gt;</code>, task_id:<code>14951 &lt;https://www.openml.org/t/14951&gt;</code></li> <li>Volcanoes on Venus: data_id:<code>1527 &lt;https://www.openml.org/d/1527&gt;</code>, task_id:<code>10103 &lt;https://www.openml.org/t/10103&gt;</code></li> <li>Walking activity: data_id:<code>1509 &lt;https://www.openml.org/d/1509&gt;</code>, task_id:<code>9945 &lt;https://www.openml.org/t/9945&gt;</code>, 150k instances.</li> <li>Covertype (Satellite): data_id:<code>150 &lt;https://www.openml.org/d/150&gt;</code>, task_id:<code>218 &lt;https://www.openml.org/t/218&gt;</code>, 500k instances.</li> <li>Higgs (Physics): data_id:<code>23512 &lt;https://www.openml.org/d/23512&gt;</code>, task_id:<code>52950 &lt;https://www.openml.org/t/52950&gt;</code>, 100k instances, missing values.</li> </ul>"},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/","title":"Plotting hyperparameter surfaces","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nimport numpy as np\n</pre> import openml import numpy as np In\u00a0[\u00a0]: Copied! <pre>df = openml.evaluations.list_evaluations_setups(\n    function=\"predictive_accuracy\",\n    flows=[8353],\n    tasks=[6],\n    output_format=\"dataframe\",\n    # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise,\n    # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary.\n    parameters_in_separate_columns=True,\n)\nprint(df.head(n=10))\n</pre> df = openml.evaluations.list_evaluations_setups(     function=\"predictive_accuracy\",     flows=[8353],     tasks=[6],     output_format=\"dataframe\",     # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise,     # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary.     parameters_in_separate_columns=True, ) print(df.head(n=10)) <p>We can see all the hyperparameter names in the columns of the dataframe:</p> In\u00a0[\u00a0]: Copied! <pre>for name in df.columns:\n    print(name)\n</pre> for name in df.columns:     print(name) <p>Next, we cast and transform the hyperparameters of interest (<code>C</code> and <code>gamma</code>) so that we can nicely plot them.</p> In\u00a0[\u00a0]: Copied! <pre>hyperparameters = [\"sklearn.svm.classes.SVC(16)_C\", \"sklearn.svm.classes.SVC(16)_gamma\"]\ndf[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10)\n</pre> hyperparameters = [\"sklearn.svm.classes.SVC(16)_C\", \"sklearn.svm.classes.SVC(16)_gamma\"] df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10) In\u00a0[\u00a0]: Copied! <pre>df.plot.hexbin(\n    x=\"sklearn.svm.classes.SVC(16)_C\",\n    y=\"sklearn.svm.classes.SVC(16)_gamma\",\n    C=\"value\",\n    reduce_C_function=np.mean,\n    gridsize=25,\n    title=\"SVM performance landscape\",\n)\n</pre> df.plot.hexbin(     x=\"sklearn.svm.classes.SVC(16)_C\",     y=\"sklearn.svm.classes.SVC(16)_gamma\",     C=\"value\",     reduce_C_function=np.mean,     gridsize=25,     title=\"SVM performance landscape\", ) In\u00a0[\u00a0]: Copied! <pre>import matplotlib.pyplot as plt\n\nfig, ax = plt.subplots()\n\nC = df[\"sklearn.svm.classes.SVC(16)_C\"]\ngamma = df[\"sklearn.svm.classes.SVC(16)_gamma\"]\nscore = df[\"value\"]\n\n# Plotting all evaluations:\nax.plot(C, gamma, \"ko\", ms=1)\n# Create a contour plot\ncntr = ax.tricontourf(C, gamma, score, levels=12, cmap=\"RdBu_r\")\n# Adjusting the colorbar\nfig.colorbar(cntr, ax=ax, label=\"accuracy\")\n# Adjusting the axis limits\nax.set(\n    xlim=(min(C), max(C)),\n    ylim=(min(gamma), max(gamma)),\n    xlabel=\"C (log10)\",\n    ylabel=\"gamma (log10)\",\n)\nax.set_title(\"SVM performance landscape\")\n# License: BSD 3-Clause\n</pre> import matplotlib.pyplot as plt  fig, ax = plt.subplots()  C = df[\"sklearn.svm.classes.SVC(16)_C\"] gamma = df[\"sklearn.svm.classes.SVC(16)_gamma\"] score = df[\"value\"]  # Plotting all evaluations: ax.plot(C, gamma, \"ko\", ms=1) # Create a contour plot cntr = ax.tricontourf(C, gamma, score, levels=12, cmap=\"RdBu_r\") # Adjusting the colorbar fig.colorbar(cntr, ax=ax, label=\"accuracy\") # Adjusting the axis limits ax.set(     xlim=(min(C), max(C)),     ylim=(min(gamma), max(gamma)),     xlabel=\"C (log10)\",     ylabel=\"gamma (log10)\", ) ax.set_title(\"SVM performance landscape\") # License: BSD 3-Clause"},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#plotting-hyperparameter-surfaces","title":"Plotting hyperparameter surfaces\u00b6","text":""},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#first-step-obtaining-the-data","title":"First step - obtaining the data\u00b6","text":"<p>First, we need to choose an SVM flow, for example 8353, and a task. Finding the IDs of them are not part of this tutorial, this could for example be done via the website.</p> <p>For this we use the function <code>list_evaluations_setup</code> which can automatically join evaluations conducted by the server with the hyperparameter settings extracted from the uploaded runs (called setup).</p>"},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#option-1-plotting-via-the-pandas-helper-functions","title":"Option 1 - plotting via the pandas helper functions\u00b6","text":""},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#option-2-plotting-via-matplotlib","title":"Option 2 - plotting via matplotlib\u00b6","text":""},{"location":"examples/30_extended/run_setup_tutorial/","title":"Run Setup","text":"In\u00a0[\u00a0]: Copied! <pre>import numpy as np\nimport openml\nfrom openml.extensions.sklearn import cat, cont\n\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder, FunctionTransformer\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.decomposition import TruncatedSVD\n</pre>  import numpy as np import openml from openml.extensions.sklearn import cat, cont  from sklearn.pipeline import make_pipeline, Pipeline from sklearn.compose import ColumnTransformer from sklearn.impute import SimpleImputer from sklearn.preprocessing import OneHotEncoder, FunctionTransformer from sklearn.ensemble import RandomForestClassifier from sklearn.decomposition import TruncatedSVD <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <ol> <li>Create a flow and use it to solve a task</li> </ol> <p>First, let's download the task that we are interested in</p> In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(6)\n</pre> task = openml.tasks.get_task(6) <p>we will create a fairly complex model, with many preprocessing components and many potential hyperparameters. Of course, the model can be as complex and as easy as you want it to be</p> In\u00a0[\u00a0]: Copied! <pre>cat_imp = make_pipeline(\n    OneHotEncoder(handle_unknown=\"ignore\", sparse=False),\n    TruncatedSVD(),\n)\ncont_imp = SimpleImputer(strategy=\"median\")\nct = ColumnTransformer([(\"cat\", cat_imp, cat), (\"cont\", cont_imp, cont)])\nmodel_original = Pipeline(\n    steps=[\n        (\"transform\", ct),\n        (\"estimator\", RandomForestClassifier()),\n    ]\n)\n</pre> cat_imp = make_pipeline(     OneHotEncoder(handle_unknown=\"ignore\", sparse=False),     TruncatedSVD(), ) cont_imp = SimpleImputer(strategy=\"median\") ct = ColumnTransformer([(\"cat\", cat_imp, cat), (\"cont\", cont_imp, cont)]) model_original = Pipeline(     steps=[         (\"transform\", ct),         (\"estimator\", RandomForestClassifier()),     ] ) <p>Let's change some hyperparameters. Of course, in any good application we would tune them using, e.g., Random Search or Bayesian Optimization, but for the purpose of this tutorial we set them to some specific values that might or might not be optimal</p> In\u00a0[\u00a0]: Copied! <pre>hyperparameters_original = {\n    \"estimator__criterion\": \"gini\",\n    \"estimator__n_estimators\": 50,\n    \"estimator__max_depth\": 10,\n    \"estimator__min_samples_leaf\": 1,\n}\nmodel_original.set_params(**hyperparameters_original)\n\n# solve the task and upload the result (this implicitly creates the flow)\nrun = openml.runs.run_model_on_task(model_original, task, avoid_duplicate_runs=False)\nrun_original = run.publish()  # this implicitly uploads the flow\n</pre> hyperparameters_original = {     \"estimator__criterion\": \"gini\",     \"estimator__n_estimators\": 50,     \"estimator__max_depth\": 10,     \"estimator__min_samples_leaf\": 1, } model_original.set_params(**hyperparameters_original)  # solve the task and upload the result (this implicitly creates the flow) run = openml.runs.run_model_on_task(model_original, task, avoid_duplicate_runs=False) run_original = run.publish()  # this implicitly uploads the flow In\u00a0[\u00a0]: Copied! <pre># obtain setup id (note that the setup id is assigned by the OpenML server -\n# therefore it was not yet available in our local copy of the run)\nrun_downloaded = openml.runs.get_run(run_original.run_id)\nsetup_id = run_downloaded.setup_id\n\n# after this, we can easily reinstantiate the model\nmodel_duplicate = openml.setups.initialize_model(setup_id)\n# it will automatically have all the hyperparameters set\n\n# and run the task again\nrun_duplicate = openml.runs.run_model_on_task(model_duplicate, task, avoid_duplicate_runs=False)\n</pre> # obtain setup id (note that the setup id is assigned by the OpenML server - # therefore it was not yet available in our local copy of the run) run_downloaded = openml.runs.get_run(run_original.run_id) setup_id = run_downloaded.setup_id  # after this, we can easily reinstantiate the model model_duplicate = openml.setups.initialize_model(setup_id) # it will automatically have all the hyperparameters set  # and run the task again run_duplicate = openml.runs.run_model_on_task(model_duplicate, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre># the run has stored all predictions in the field data content\nnp.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content)\n</pre> # the run has stored all predictions in the field data content np.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n\n# By: Jan N. van Rijn\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example()  # By: Jan N. van Rijn # License: BSD 3-Clause"},{"location":"examples/30_extended/run_setup_tutorial/#run-setup","title":"Run Setup\u00b6","text":"<p>One of the key features of the openml-python library is that is allows to reinstantiate flows with hyperparameter settings that were uploaded before. This tutorial uses the concept of setups. Although setups are not extensively described in the OpenML documentation (because most users will not directly use them), they form a important concept within OpenML distinguishing between hyperparameter configurations. A setup is the combination of a flow with all its hyperparameters set.</p> <p>A key requirement for reinstantiating a flow is to have the same scikit-learn version as the flow that was uploaded. However, this tutorial will upload the flow (that will later be reinstantiated) itself, so it can be ran with any scikit-learn version that is supported by this library. In this case, the requirement of the corresponding scikit-learn versions is automatically met.</p> <p>In this tutorial we will 1) Create a flow and use it to solve a task; 2) Download the flow, reinstantiate the model with same hyperparameters, and solve the same task again; 3) We will verify that the obtained results are exactly the same.</p>"},{"location":"examples/30_extended/run_setup_tutorial/#2-download-the-flow-and-solve-the-same-task-again","title":"2) Download the flow and solve the same task again.\u00b6","text":""},{"location":"examples/30_extended/run_setup_tutorial/#3-we-will-verify-that-the-obtained-results-are-exactly-the-same","title":"3) We will verify that the obtained results are exactly the same.\u00b6","text":""},{"location":"examples/30_extended/study_tutorial/","title":"Benchmark studies","text":"In\u00a0[\u00a0]: Copied! <pre>import uuid\n\nfrom sklearn.ensemble import RandomForestClassifier\n\nimport openml\n</pre> import uuid  from sklearn.ensemble import RandomForestClassifier  import openml In\u00a0[\u00a0]: Copied! <pre>studies = openml.study.list_studies(output_format=\"dataframe\", status=\"all\")\nprint(studies.head(n=10))\n</pre> studies = openml.study.list_studies(output_format=\"dataframe\", status=\"all\") print(studies.head(n=10)) <p>This is done based on the study ID.</p> In\u00a0[\u00a0]: Copied! <pre>study = openml.study.get_study(123)\nprint(study)\n</pre> study = openml.study.get_study(123) print(study) <p>Studies also features a description:</p> In\u00a0[\u00a0]: Copied! <pre>print(study.description)\n</pre> print(study.description) <p>Studies are a container for runs:</p> In\u00a0[\u00a0]: Copied! <pre>print(study.runs)\n</pre> print(study.runs) <p>And we can use the evaluation listing functionality to learn more about the evaluations available for the conducted runs:</p> In\u00a0[\u00a0]: Copied! <pre>evaluations = openml.evaluations.list_evaluations(\n    function=\"predictive_accuracy\",\n    output_format=\"dataframe\",\n    study=study.study_id,\n)\nprint(evaluations.head())\n</pre> evaluations = openml.evaluations.list_evaluations(     function=\"predictive_accuracy\",     output_format=\"dataframe\",     study=study.study_id, ) print(evaluations.head()) <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre># Model to be used\nclf = RandomForestClassifier()\n\n# We'll create a study with one run on 3 datasets present in the suite\ntasks = [115, 259, 307]\n\n# To verify\nsuite = openml.study.get_suite(1)\nprint(all([t_id in suite.tasks for t_id in tasks]))\n\nrun_ids = []\nfor task_id in tasks:\n    task = openml.tasks.get_task(task_id)\n    run = openml.runs.run_model_on_task(clf, task)\n    run.publish()\n    run_ids.append(run.run_id)\n\n# The study needs a machine-readable and unique alias. To obtain this,\n# we simply generate a random uuid.\nalias = uuid.uuid4().hex\n\nnew_study = openml.study.create_study(\n    name=\"Test-Study\",\n    description=\"Test study for the Python tutorial on studies\",\n    run_ids=run_ids,\n    alias=alias,\n    benchmark_suite=suite.study_id,\n)\nnew_study.publish()\nprint(new_study)\n</pre> # Model to be used clf = RandomForestClassifier()  # We'll create a study with one run on 3 datasets present in the suite tasks = [115, 259, 307]  # To verify suite = openml.study.get_suite(1) print(all([t_id in suite.tasks for t_id in tasks]))  run_ids = [] for task_id in tasks:     task = openml.tasks.get_task(task_id)     run = openml.runs.run_model_on_task(clf, task)     run.publish()     run_ids.append(run.run_id)  # The study needs a machine-readable and unique alias. To obtain this, # we simply generate a random uuid. alias = uuid.uuid4().hex  new_study = openml.study.create_study(     name=\"Test-Study\",     description=\"Test study for the Python tutorial on studies\",     run_ids=run_ids,     alias=alias,     benchmark_suite=suite.study_id, ) new_study.publish() print(new_study) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/study_tutorial/#benchmark-studies","title":"Benchmark studies\u00b6","text":"<p>How to list, download and upload benchmark studies. In contrast to benchmark suites which hold a list of tasks, studies hold a list of runs. As runs contain all information on flows and tasks, all required information about a study can be retrieved.</p>"},{"location":"examples/30_extended/study_tutorial/#listing-studies","title":"Listing studies\u00b6","text":"<ul> <li>Use the output_format parameter to select output type</li> <li>Default gives <code>dict</code>, but we'll use <code>dataframe</code> to obtain an easier-to-work-with data structure</li> </ul>"},{"location":"examples/30_extended/study_tutorial/#downloading-studies","title":"Downloading studies\u00b6","text":""},{"location":"examples/30_extended/study_tutorial/#uploading-studies","title":"Uploading studies\u00b6","text":"<p>Creating a study is as simple as creating any kind of other OpenML entity. In this examples we'll create a few runs for the OpenML-100 benchmark suite which is available on the OpenML test server.</p>"},{"location":"examples/30_extended/suites_tutorial/","title":"Benchmark suites","text":"In\u00a0[\u00a0]: Copied! <pre>import uuid\n\nimport numpy as np\n\nimport openml\n</pre> import uuid  import numpy as np  import openml In\u00a0[\u00a0]: Copied! <pre>suites = openml.study.list_suites(output_format=\"dataframe\", status=\"all\")\nprint(suites.head(n=10))\n</pre> suites = openml.study.list_suites(output_format=\"dataframe\", status=\"all\") print(suites.head(n=10)) <p>This is done based on the dataset ID.</p> In\u00a0[\u00a0]: Copied! <pre>suite = openml.study.get_suite(99)\nprint(suite)\n</pre> suite = openml.study.get_suite(99) print(suite) <p>Suites also feature a description:</p> In\u00a0[\u00a0]: Copied! <pre>print(suite.description)\n</pre> print(suite.description) <p>Suites are a container for tasks:</p> In\u00a0[\u00a0]: Copied! <pre>print(suite.tasks)\n</pre> print(suite.tasks) <p>And we can use the task listing functionality to learn more about them:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(output_format=\"dataframe\")\n</pre> tasks = openml.tasks.list_tasks(output_format=\"dataframe\") <p>Using <code>@</code> in pd.DataFrame.query accesses variables outside of the current dataframe.</p> In\u00a0[\u00a0]: Copied! <pre>tasks = tasks.query(\"tid in @suite.tasks\")\nprint(tasks.describe().transpose())\n</pre> tasks = tasks.query(\"tid in @suite.tasks\") print(tasks.describe().transpose()) <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>all_tasks = list(openml.tasks.list_tasks(output_format=\"dataframe\")[\"tid\"])\ntask_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20))\n\n# The study needs a machine-readable and unique alias. To obtain this,\n# we simply generate a random uuid.\n\nalias = uuid.uuid4().hex\n\nnew_suite = openml.study.create_benchmark_suite(\n    name=\"Test-Suite\",\n    description=\"Test suite for the Python tutorial on benchmark suites\",\n    task_ids=task_ids_for_suite,\n    alias=alias,\n)\nnew_suite.publish()\nprint(new_suite)\n</pre> all_tasks = list(openml.tasks.list_tasks(output_format=\"dataframe\")[\"tid\"]) task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20))  # The study needs a machine-readable and unique alias. To obtain this, # we simply generate a random uuid.  alias = uuid.uuid4().hex  new_suite = openml.study.create_benchmark_suite(     name=\"Test-Suite\",     description=\"Test suite for the Python tutorial on benchmark suites\",     task_ids=task_ids_for_suite,     alias=alias, ) new_suite.publish() print(new_suite) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/suites_tutorial/#benchmark-suites","title":"Benchmark suites\u00b6","text":"<p>How to list, download and upload benchmark suites.</p> <p>If you want to learn more about benchmark suites, check out our brief introductory tutorial \"Simple suites tutorial\" or the OpenML benchmark docs.</p>"},{"location":"examples/30_extended/suites_tutorial/#listing-suites","title":"Listing suites\u00b6","text":"<ul> <li>Use the output_format parameter to select output type</li> <li>Default gives <code>dict</code>, but we'll use <code>dataframe</code> to obtain an easier-to-work-with data structure</li> </ul>"},{"location":"examples/30_extended/suites_tutorial/#downloading-suites","title":"Downloading suites\u00b6","text":""},{"location":"examples/30_extended/suites_tutorial/#uploading-suites","title":"Uploading suites\u00b6","text":"<p>Uploading suites is as simple as uploading any kind of other OpenML entity - the only reason why we need so much code in this example is because we upload some random data.</p> <p>We'll take a random subset of at least ten tasks of all available tasks on the test server:</p>"},{"location":"examples/30_extended/task_manual_iteration_tutorial/","title":"Tasks: retrieving splits","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre> import openml <p>For this tutorial we will use the famous King+Rook versus King+Pawn on A7 dataset, which has the dataset ID 3 (dataset on OpenML), and for which there exist tasks with all important estimation procedures. It is small enough (less than 5000 samples) to efficiently use it in an example.</p> <p>We will first start with (task 233), which is a task with a holdout estimation procedure.</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 233\ntask = openml.tasks.get_task(task_id)\n</pre> task_id = 233 task = openml.tasks.get_task(task_id) <p>Now that we have a task object we can obtain the number of repetitions, folds and samples as defined by the task:</p> In\u00a0[\u00a0]: Copied! <pre>n_repeats, n_folds, n_samples = task.get_split_dimensions()\n</pre> n_repeats, n_folds, n_samples = task.get_split_dimensions() <ul> <li><code>n_repeats</code>: Number of times the model quality estimation is performed</li> <li><code>n_folds</code>: Number of folds per repeat</li> <li><code>n_samples</code>: How many data points to use. This is only relevant for learning curve tasks</li> </ul> <p>A list of all available estimation procedures is available here.</p> <p>Task <code>233</code> is a simple task using the holdout estimation procedure and therefore has only a single repeat, a single fold and a single sample size:</p> In\u00a0[\u00a0]: Copied! <pre>print(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>We can now retrieve the train/test split for this combination of repeats, folds and number of samples (indexing is zero-based). Usually, one would loop over all repeats, folds and sample sizes, but we can neglect this here as there is only a single repetition.</p> In\u00a0[\u00a0]: Copied! <pre>train_indices, test_indices = task.get_train_test_split_indices(\n    repeat=0,\n    fold=0,\n    sample=0,\n)\n\nprint(train_indices.shape, train_indices.dtype)\nprint(test_indices.shape, test_indices.dtype)\n</pre> train_indices, test_indices = task.get_train_test_split_indices(     repeat=0,     fold=0,     sample=0, )  print(train_indices.shape, train_indices.dtype) print(test_indices.shape, test_indices.dtype) <p>And then split the data based on this:</p> In\u00a0[\u00a0]: Copied! <pre>X, y = task.get_X_and_y(dataset_format=\"dataframe\")\nX_train = X.iloc[train_indices]\ny_train = y.iloc[train_indices]\nX_test = X.iloc[test_indices]\ny_test = y.iloc[test_indices]\n\nprint(\n    \"X_train.shape: {}, y_train.shape: {}, X_test.shape: {}, y_test.shape: {}\".format(\n        X_train.shape,\n        y_train.shape,\n        X_test.shape,\n        y_test.shape,\n    )\n)\n</pre> X, y = task.get_X_and_y(dataset_format=\"dataframe\") X_train = X.iloc[train_indices] y_train = y.iloc[train_indices] X_test = X.iloc[test_indices] y_test = y.iloc[test_indices]  print(     \"X_train.shape: {}, y_train.shape: {}, X_test.shape: {}, y_test.shape: {}\".format(         X_train.shape,         y_train.shape,         X_test.shape,         y_test.shape,     ) ) <p>Obviously, we can also retrieve cross-validation versions of the dataset used in task <code>233</code>:</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 3\ntask = openml.tasks.get_task(task_id)\nX, y = task.get_X_and_y(dataset_format=\"dataframe\")\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> task_id = 3 task = openml.tasks.get_task(task_id) X, y = task.get_X_and_y(dataset_format=\"dataframe\") n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>And then perform the aforementioned iteration over all splits:</p> In\u00a0[\u00a0]: Copied! <pre>for repeat_idx in range(n_repeats):\n    for fold_idx in range(n_folds):\n        for sample_idx in range(n_samples):\n            train_indices, test_indices = task.get_train_test_split_indices(\n                repeat=repeat_idx,\n                fold=fold_idx,\n                sample=sample_idx,\n            )\n            X_train = X.iloc[train_indices]\n            y_train = y.iloc[train_indices]\n            X_test = X.iloc[test_indices]\n            y_test = y.iloc[test_indices]\n\n            print(\n                \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"\n                \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(\n                    repeat_idx,\n                    fold_idx,\n                    sample_idx,\n                    X_train.shape,\n                    y_train.shape,\n                    X_test.shape,\n                    y_test.shape,\n                )\n            )\n</pre> for repeat_idx in range(n_repeats):     for fold_idx in range(n_folds):         for sample_idx in range(n_samples):             train_indices, test_indices = task.get_train_test_split_indices(                 repeat=repeat_idx,                 fold=fold_idx,                 sample=sample_idx,             )             X_train = X.iloc[train_indices]             y_train = y.iloc[train_indices]             X_test = X.iloc[test_indices]             y_test = y.iloc[test_indices]              print(                 \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"                 \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(                     repeat_idx,                     fold_idx,                     sample_idx,                     X_train.shape,                     y_train.shape,                     X_test.shape,                     y_test.shape,                 )             ) <p>And also versions with multiple repeats:</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 1767\ntask = openml.tasks.get_task(task_id)\nX, y = task.get_X_and_y(dataset_format=\"dataframe\")\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> task_id = 1767 task = openml.tasks.get_task(task_id) X, y = task.get_X_and_y(dataset_format=\"dataframe\") n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>And then again perform the aforementioned iteration over all splits:</p> In\u00a0[\u00a0]: Copied! <pre>for repeat_idx in range(n_repeats):\n    for fold_idx in range(n_folds):\n        for sample_idx in range(n_samples):\n            train_indices, test_indices = task.get_train_test_split_indices(\n                repeat=repeat_idx,\n                fold=fold_idx,\n                sample=sample_idx,\n            )\n            X_train = X.iloc[train_indices]\n            y_train = y.iloc[train_indices]\n            X_test = X.iloc[test_indices]\n            y_test = y.iloc[test_indices]\n\n            print(\n                \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"\n                \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(\n                    repeat_idx,\n                    fold_idx,\n                    sample_idx,\n                    X_train.shape,\n                    y_train.shape,\n                    X_test.shape,\n                    y_test.shape,\n                )\n            )\n</pre> for repeat_idx in range(n_repeats):     for fold_idx in range(n_folds):         for sample_idx in range(n_samples):             train_indices, test_indices = task.get_train_test_split_indices(                 repeat=repeat_idx,                 fold=fold_idx,                 sample=sample_idx,             )             X_train = X.iloc[train_indices]             y_train = y.iloc[train_indices]             X_test = X.iloc[test_indices]             y_test = y.iloc[test_indices]              print(                 \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"                 \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(                     repeat_idx,                     fold_idx,                     sample_idx,                     X_train.shape,                     y_train.shape,                     X_test.shape,                     y_test.shape,                 )             ) <p>And finally a task based on learning curves:</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 1702\ntask = openml.tasks.get_task(task_id)\nX, y = task.get_X_and_y(dataset_format=\"dataframe\")\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> task_id = 1702 task = openml.tasks.get_task(task_id) X, y = task.get_X_and_y(dataset_format=\"dataframe\") n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>And then again perform the aforementioned iteration over all splits:</p> In\u00a0[\u00a0]: Copied! <pre>for repeat_idx in range(n_repeats):\n    for fold_idx in range(n_folds):\n        for sample_idx in range(n_samples):\n            train_indices, test_indices = task.get_train_test_split_indices(\n                repeat=repeat_idx,\n                fold=fold_idx,\n                sample=sample_idx,\n            )\n            X_train = X.iloc[train_indices]\n            y_train = y.iloc[train_indices]\n            X_test = X.iloc[test_indices]\n            y_test = y.iloc[test_indices]\n\n            print(\n                \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"\n                \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(\n                    repeat_idx,\n                    fold_idx,\n                    sample_idx,\n                    X_train.shape,\n                    y_train.shape,\n                    X_test.shape,\n                    y_test.shape,\n                )\n            )\n# License: BSD 3-Clause\n</pre> for repeat_idx in range(n_repeats):     for fold_idx in range(n_folds):         for sample_idx in range(n_samples):             train_indices, test_indices = task.get_train_test_split_indices(                 repeat=repeat_idx,                 fold=fold_idx,                 sample=sample_idx,             )             X_train = X.iloc[train_indices]             y_train = y.iloc[train_indices]             X_test = X.iloc[test_indices]             y_test = y.iloc[test_indices]              print(                 \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"                 \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(                     repeat_idx,                     fold_idx,                     sample_idx,                     X_train.shape,                     y_train.shape,                     X_test.shape,                     y_test.shape,                 )             ) # License: BSD 3-Clause"},{"location":"examples/30_extended/task_manual_iteration_tutorial/#tasks-retrieving-splits","title":"Tasks: retrieving splits\u00b6","text":"<p>Tasks define a target and a train/test split. Normally, they are the input to the function <code>openml.runs.run_model_on_task</code> which automatically runs the model on all splits of the task. However, sometimes it is necessary to manually split a dataset to perform experiments outside of the functions provided by OpenML. One such example is in the benchmark library HPOBench which extensively uses data from OpenML, but not OpenML's functionality to conduct runs.</p>"},{"location":"examples/30_extended/tasks_tutorial/","title":"Tasks","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom openml.tasks import TaskType\nimport pandas as pd\n</pre> import openml from openml.tasks import TaskType import pandas as pd <p>Tasks are identified by IDs and can be accessed in two different ways:</p> <ol> <li>In a list providing basic information on all tasks available on OpenML. This function will not download the actual tasks, but will instead download meta data that can be used to filter the tasks and retrieve a set of IDs. We can filter this list, for example, we can only list tasks having a special tag or only tasks for a specific target such as supervised classification.</li> <li>A single task by its ID. It contains all meta information, the target metric, the splits and an iterator which can be used to access the splits in a useful manner.</li> </ol> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(\n    task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format=\"dataframe\"\n)\nprint(tasks.columns)\nprint(f\"First 5 of {len(tasks)} tasks:\")\nprint(tasks.head())\n</pre> tasks = openml.tasks.list_tasks(     task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format=\"dataframe\" ) print(tasks.columns) print(f\"First 5 of {len(tasks)} tasks:\") print(tasks.head()) <p>We can filter the list of tasks to only contain datasets with more than 500 samples, but less than 1000 samples:</p> In\u00a0[\u00a0]: Copied! <pre>filtered_tasks = tasks.query(\"NumberOfInstances &gt; 500 and NumberOfInstances &lt; 1000\")\nprint(list(filtered_tasks.index))\n</pre> filtered_tasks = tasks.query(\"NumberOfInstances &gt; 500 and NumberOfInstances &lt; 1000\") print(list(filtered_tasks.index)) In\u00a0[\u00a0]: Copied! <pre># Number of tasks\nprint(len(filtered_tasks))\n</pre> # Number of tasks print(len(filtered_tasks)) <p>Then, we can further restrict the tasks to all have the same resampling strategy:</p> In\u00a0[\u00a0]: Copied! <pre>filtered_tasks = filtered_tasks.query('estimation_procedure == \"10-fold Crossvalidation\"')\nprint(list(filtered_tasks.index))\n</pre> filtered_tasks = filtered_tasks.query('estimation_procedure == \"10-fold Crossvalidation\"') print(list(filtered_tasks.index)) In\u00a0[\u00a0]: Copied! <pre># Number of tasks\nprint(len(filtered_tasks))\n</pre> # Number of tasks print(len(filtered_tasks)) <p>Resampling strategies can be found on the OpenML Website.</p> <p>Similar to listing tasks by task type, we can list tasks by tags:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(tag=\"OpenML100\", output_format=\"dataframe\")\nprint(f\"First 5 of {len(tasks)} tasks:\")\nprint(tasks.head())\n</pre> tasks = openml.tasks.list_tasks(tag=\"OpenML100\", output_format=\"dataframe\") print(f\"First 5 of {len(tasks)} tasks:\") print(tasks.head()) <p>Furthermore, we can list tasks based on the dataset id:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(data_id=1471, output_format=\"dataframe\")\nprint(f\"First 5 of {len(tasks)} tasks:\")\nprint(tasks.head())\n</pre> tasks = openml.tasks.list_tasks(data_id=1471, output_format=\"dataframe\") print(f\"First 5 of {len(tasks)} tasks:\") print(tasks.head()) <p>In addition, a size limit and an offset can be applied both separately and simultaneously:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(size=10, offset=50, output_format=\"dataframe\")\nprint(tasks)\n</pre> tasks = openml.tasks.list_tasks(size=10, offset=50, output_format=\"dataframe\") print(tasks) <p>OpenML 100 is a curated list of 100 tasks to start using OpenML. They are all supervised classification tasks with more than 500 instances and less than 50000 instances per task. To make things easier, the tasks do not contain highly unbalanced data and sparse data. However, the tasks include missing values and categorical features. You can find out more about the OpenML 100 on the OpenML benchmarking page.</p> <p>Finally, it is also possible to list all tasks on OpenML with:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(output_format=\"dataframe\")\nprint(len(tasks))\n</pre> tasks = openml.tasks.list_tasks(output_format=\"dataframe\") print(len(tasks)) In\u00a0[\u00a0]: Copied! <pre>tasks.query('name==\"eeg-eye-state\"')\n</pre> tasks.query('name==\"eeg-eye-state\"') In\u00a0[\u00a0]: Copied! <pre>task_id = 31\ntask = openml.tasks.get_task(task_id)\n</pre> task_id = 31 task = openml.tasks.get_task(task_id) In\u00a0[\u00a0]: Copied! <pre># Properties of the task are stored as member variables:\nprint(task)\n</pre> # Properties of the task are stored as member variables: print(task) In\u00a0[\u00a0]: Copied! <pre># And:\n\nids = [2, 1891, 31, 9983]\ntasks = openml.tasks.get_tasks(ids)\nprint(tasks[0])\n</pre> # And:  ids = [2, 1891, 31, 9983] tasks = openml.tasks.get_tasks(ids) print(tasks[0]) <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>try:\n    my_task = openml.tasks.create_task(\n        task_type=TaskType.SUPERVISED_CLASSIFICATION,\n        dataset_id=128,\n        target_name=\"class\",\n        evaluation_measure=\"predictive_accuracy\",\n        estimation_procedure_id=1,\n    )\n    my_task.publish()\nexcept openml.exceptions.OpenMLServerException as e:\n    # Error code for 'task already exists'\n    if e.code == 614:\n        # Lookup task\n        tasks = openml.tasks.list_tasks(data_id=128, output_format=\"dataframe\")\n        tasks = tasks.query(\n            'task_type == \"Supervised Classification\" '\n            'and estimation_procedure == \"10-fold Crossvalidation\" '\n            'and evaluation_measures == \"predictive_accuracy\"'\n        )\n        task_id = tasks.loc[:, \"tid\"].values[0]\n        print(\"Task already exists. Task ID is\", task_id)\n</pre> try:     my_task = openml.tasks.create_task(         task_type=TaskType.SUPERVISED_CLASSIFICATION,         dataset_id=128,         target_name=\"class\",         evaluation_measure=\"predictive_accuracy\",         estimation_procedure_id=1,     )     my_task.publish() except openml.exceptions.OpenMLServerException as e:     # Error code for 'task already exists'     if e.code == 614:         # Lookup task         tasks = openml.tasks.list_tasks(data_id=128, output_format=\"dataframe\")         tasks = tasks.query(             'task_type == \"Supervised Classification\" '             'and estimation_procedure == \"10-fold Crossvalidation\" '             'and evaluation_measures == \"predictive_accuracy\"'         )         task_id = tasks.loc[:, \"tid\"].values[0]         print(\"Task already exists. Task ID is\", task_id) In\u00a0[\u00a0]: Copied! <pre># reverting to prod server\nopenml.config.stop_using_configuration_for_example()\n</pre> # reverting to prod server openml.config.stop_using_configuration_for_example() <ul> <li>Complete list of task types.</li> <li>Complete list of model estimation procedures.</li> <li>Complete list of evaluation measures.</li> </ul> <p>License: BSD 3-Clause</p>"},{"location":"examples/30_extended/tasks_tutorial/#tasks","title":"Tasks\u00b6","text":"<p>A tutorial on how to list and download tasks.</p>"},{"location":"examples/30_extended/tasks_tutorial/#listing-tasks","title":"Listing tasks\u00b6","text":"<p>We will start by simply listing only supervised classification tasks.</p> <p>openml.tasks.list_tasks() returns a dictionary of dictionaries by default, but we request a pandas dataframe instead to have better visualization capabilities and easier access:</p>"},{"location":"examples/30_extended/tasks_tutorial/#exercise","title":"Exercise\u00b6","text":"<p>Search for the tasks on the 'eeg-eye-state' dataset.</p>"},{"location":"examples/30_extended/tasks_tutorial/#downloading-tasks","title":"Downloading tasks\u00b6","text":"<p>We provide two functions to download tasks, one which downloads only a single task by its ID, and one which takes a list of IDs and downloads all of these tasks:</p>"},{"location":"examples/30_extended/tasks_tutorial/#creating-tasks","title":"Creating tasks\u00b6","text":"<p>You can also create new tasks. Take the following into account:</p> <ul> <li>You can only create tasks on active datasets</li> <li>For now, only the following tasks are supported: classification, regression, clustering, and learning curve analysis.</li> <li>For now, tasks can only be created on a single dataset.</li> <li>The exact same task must not already exist.</li> </ul> <p>Creating a task requires the following input:</p> <ul> <li>task_type: The task type ID, required (see below). Required.</li> <li>dataset_id: The dataset ID. Required.</li> <li>target_name: The name of the attribute you aim to predict. Optional.</li> <li>estimation_procedure_id : The ID of the estimation procedure used to create train-test splits. Optional.</li> <li>evaluation_measure: The name of the evaluation measure. Optional.</li> <li>Any additional inputs for specific tasks</li> </ul> <p>It is best to leave the evaluation measure open if there is no strong prerequisite for a specific measure. OpenML will always compute all appropriate measures and you can filter or sort results on your favourite measure afterwards. Only add an evaluation measure if necessary (e.g. when other measure make no sense), since it will create a new task, which scatters results across tasks.</p>"},{"location":"examples/30_extended/tasks_tutorial/#example","title":"Example\u00b6","text":"<p>Let's create a classification task on a dataset. In this example we will do this on the Iris dataset (ID=128 (on test server)). We'll use 10-fold cross-validation (ID=1), and predictive accuracy as the predefined measure (this can also be left open). If a task with these parameters exists, we will get an appropriate exception. If such a task doesn't exist, a task will be created and the corresponding task_id will be returned.</p>"},{"location":"examples/40_paper/2015_neurips_feurer_example/","title":"Feurer et al. (2015)","text":"In\u00a0[\u00a0]: Copied! <pre>import pandas as pd\n\nimport openml\n</pre> import pandas as pd  import openml <p>List of dataset IDs given in the supplementary material of Feurer et al.: https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip</p> In\u00a0[\u00a0]: Copied! <pre>dataset_ids = [\n    3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,\n    57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,\n    390, 391, 392, 393, 395, 396, 398, 399, 401, 554, 679, 715, 718, 720, 722,\n    723, 727, 728, 734, 735, 737, 740, 741, 743, 751, 752, 761, 772, 797, 799,\n    803, 806, 807, 813, 816, 819, 821, 822, 823, 833, 837, 843, 845, 846, 847,\n    849, 866, 871, 881, 897, 901, 903, 904, 910, 912, 913, 914, 917, 923, 930,\n    934, 953, 958, 959, 962, 966, 971, 976, 977, 978, 979, 980, 991, 993, 995,\n    1000, 1002, 1018, 1019, 1020, 1021, 1036, 1040, 1041, 1049, 1050, 1053,\n    1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,\n    1134, 1138, 1139, 1142, 1146, 1161, 1166,\n]\n</pre> dataset_ids = [     3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,     57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,     390, 391, 392, 393, 395, 396, 398, 399, 401, 554, 679, 715, 718, 720, 722,     723, 727, 728, 734, 735, 737, 740, 741, 743, 751, 752, 761, 772, 797, 799,     803, 806, 807, 813, 816, 819, 821, 822, 823, 833, 837, 843, 845, 846, 847,     849, 866, 871, 881, 897, 901, 903, 904, 910, 912, 913, 914, 917, 923, 930,     934, 953, 958, 959, 962, 966, 971, 976, 977, 978, 979, 980, 991, 993, 995,     1000, 1002, 1018, 1019, 1020, 1021, 1036, 1040, 1041, 1049, 1050, 1053,     1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,     1134, 1138, 1139, 1142, 1146, 1161, 1166, ] <p>The dataset IDs could be used directly to load the dataset and split the data into a training set and a test set. However, to be reproducible, we will first obtain the respective tasks from OpenML, which define both the target feature and the train/test split.</p> <p>.. note:: It is discouraged to work directly on datasets and only provide dataset IDs in a paper as this does not allow reproducibility (unclear splitting). Please do not use datasets but the respective tasks as basis for a paper and publish task IDS. This example is only given to showcase the use of OpenML-Python for a published paper and as a warning on how not to do it. Please check the <code>OpenML documentation of tasks &lt;https://docs.openml.org/#tasks&gt;</code>_ if you want to learn more about them.</p> <p>This lists both active and inactive tasks (because of <code>status='all'</code>). Unfortunately, this is necessary as some of the datasets contain issues found after the publication and became deactivated, which also deactivated the tasks on them. More information on active or inactive datasets can be found in the online docs.</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(\n    task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION,\n    status=\"all\",\n    output_format=\"dataframe\",\n)\n\n# Query only those with holdout as the resampling startegy.\ntasks = tasks.query('estimation_procedure == \"33% Holdout set\"')\n\ntask_ids = []\nfor did in dataset_ids:\n    tasks_ = list(tasks.query(\"did == {}\".format(did)).tid)\n    if len(tasks_) &gt;= 1:  # if there are multiple task, take the one with lowest ID (oldest).\n        task_id = min(tasks_)\n    else:\n        raise ValueError(did)\n\n    # Optional - Check that the task has the same target attribute as the\n    # dataset default target attribute\n    # (disabled for this example as it needs to run fast to be rendered online)\n    # task = openml.tasks.get_task(task_id)\n    # dataset = task.get_dataset()\n    # if task.target_name != dataset.default_target_attribute:\n    #     raise ValueError(\n    #         (task.target_name, dataset.default_target_attribute)\n    #     )\n\n    task_ids.append(task_id)\n\nassert len(task_ids) == 140\ntask_ids.sort()\n\n# These are the tasks to work with:\nprint(task_ids)\n\n# License: BSD 3-Clause\n</pre> tasks = openml.tasks.list_tasks(     task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION,     status=\"all\",     output_format=\"dataframe\", )  # Query only those with holdout as the resampling startegy. tasks = tasks.query('estimation_procedure == \"33% Holdout set\"')  task_ids = [] for did in dataset_ids:     tasks_ = list(tasks.query(\"did == {}\".format(did)).tid)     if len(tasks_) &gt;= 1:  # if there are multiple task, take the one with lowest ID (oldest).         task_id = min(tasks_)     else:         raise ValueError(did)      # Optional - Check that the task has the same target attribute as the     # dataset default target attribute     # (disabled for this example as it needs to run fast to be rendered online)     # task = openml.tasks.get_task(task_id)     # dataset = task.get_dataset()     # if task.target_name != dataset.default_target_attribute:     #     raise ValueError(     #         (task.target_name, dataset.default_target_attribute)     #     )      task_ids.append(task_id)  assert len(task_ids) == 140 task_ids.sort()  # These are the tasks to work with: print(task_ids)  # License: BSD 3-Clause"},{"location":"examples/40_paper/2015_neurips_feurer_example/#feurer-et-al-2015","title":"Feurer et al. (2015)\u00b6","text":"<p>A tutorial on how to get the datasets used in the paper introducing Auto-sklearn by Feurer et al..</p> <p>Auto-sklearn website: https://automl.github.io/auto-sklearn/</p>"},{"location":"examples/40_paper/2015_neurips_feurer_example/#publication","title":"Publication\u00b6","text":"<p>| Efficient and Robust Automated Machine Learning | Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter | In Advances in Neural Information Processing Systems 28, 2015 | Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf</p>"},{"location":"examples/40_paper/2018_ida_strang_example/","title":"Strang et al. (2018)","text":"In\u00a0[\u00a0]: Copied! <pre>import matplotlib.pyplot as plt\nimport openml\nimport pandas as pd\n</pre> import matplotlib.pyplot as plt import openml import pandas as pd <p>A basic step for each data-mining or machine learning task is to determine which model to choose based on the problem and the data at hand. In this work we investigate when non-linear classifiers outperform linear classifiers by means of a large scale experiment.</p> <p>The paper is accompanied with a study object, containing all relevant tasks and runs (<code>study_id=123</code>). The paper features three experiment classes: Support Vector Machines (SVM), Neural Networks (NN) and Decision Trees (DT). This example demonstrates how to reproduce the plots, comparing two classifiers given the OpenML flow ids. Note that this allows us to reproduce the SVM and NN experiment, but not the DT experiment, as this requires a bit more effort to distinguish the same flow with different hyperparameter values.</p> In\u00a0[\u00a0]: Copied! <pre>study_id = 123\n# for comparing svms: flow_ids = [7754, 7756]\n# for comparing nns: flow_ids = [7722, 7729]\n# for comparing dts: flow_ids = [7725], differentiate on hyper-parameter value\nclassifier_family = \"SVM\"\nflow_ids = [7754, 7756]\nmeasure = \"predictive_accuracy\"\nmeta_features = [\"NumberOfInstances\", \"NumberOfFeatures\"]\nclass_values = [\"non-linear better\", \"linear better\", \"equal\"]\n\n# Downloads all evaluation records related to this study\nevaluations = openml.evaluations.list_evaluations(\n    measure, size=None, flows=flow_ids, study=study_id, output_format=\"dataframe\"\n)\n# gives us a table with columns data_id, flow1_value, flow2_value\nevaluations = evaluations.pivot(index=\"data_id\", columns=\"flow_id\", values=\"value\").dropna()\n# downloads all data qualities (for scatter plot)\ndata_qualities = openml.datasets.list_datasets(\n    data_id=list(evaluations.index.values), output_format=\"dataframe\"\n)\n# removes irrelevant data qualities\ndata_qualities = data_qualities[meta_features]\n# makes a join between evaluation table and data qualities table,\n# now we have columns data_id, flow1_value, flow2_value, meta_feature_1,\n# meta_feature_2\nevaluations = evaluations.join(data_qualities, how=\"inner\")\n\n# adds column that indicates the difference between the two classifiers\nevaluations[\"diff\"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]]\n</pre> study_id = 123 # for comparing svms: flow_ids = [7754, 7756] # for comparing nns: flow_ids = [7722, 7729] # for comparing dts: flow_ids = [7725], differentiate on hyper-parameter value classifier_family = \"SVM\" flow_ids = [7754, 7756] measure = \"predictive_accuracy\" meta_features = [\"NumberOfInstances\", \"NumberOfFeatures\"] class_values = [\"non-linear better\", \"linear better\", \"equal\"]  # Downloads all evaluation records related to this study evaluations = openml.evaluations.list_evaluations(     measure, size=None, flows=flow_ids, study=study_id, output_format=\"dataframe\" ) # gives us a table with columns data_id, flow1_value, flow2_value evaluations = evaluations.pivot(index=\"data_id\", columns=\"flow_id\", values=\"value\").dropna() # downloads all data qualities (for scatter plot) data_qualities = openml.datasets.list_datasets(     data_id=list(evaluations.index.values), output_format=\"dataframe\" ) # removes irrelevant data qualities data_qualities = data_qualities[meta_features] # makes a join between evaluation table and data qualities table, # now we have columns data_id, flow1_value, flow2_value, meta_feature_1, # meta_feature_2 evaluations = evaluations.join(data_qualities, how=\"inner\")  # adds column that indicates the difference between the two classifiers evaluations[\"diff\"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]] <p>makes the s-plot</p> In\u00a0[\u00a0]: Copied! <pre>fig_splot, ax_splot = plt.subplots()\nax_splot.plot(range(len(evaluations)), sorted(evaluations[\"diff\"]))\nax_splot.set_title(classifier_family)\nax_splot.set_xlabel(\"Dataset (sorted)\")\nax_splot.set_ylabel(\"difference between linear and non-linear classifier\")\nax_splot.grid(linestyle=\"--\", axis=\"y\")\nplt.show()\n</pre> fig_splot, ax_splot = plt.subplots() ax_splot.plot(range(len(evaluations)), sorted(evaluations[\"diff\"])) ax_splot.set_title(classifier_family) ax_splot.set_xlabel(\"Dataset (sorted)\") ax_splot.set_ylabel(\"difference between linear and non-linear classifier\") ax_splot.grid(linestyle=\"--\", axis=\"y\") plt.show() <p>adds column that indicates the difference between the two classifiers, needed for the scatter plot</p> In\u00a0[\u00a0]: Copied! <pre>def determine_class(val_lin, val_nonlin):\n    if val_lin &lt; val_nonlin:\n        return class_values[0]\n    elif val_nonlin &lt; val_lin:\n        return class_values[1]\n    else:\n        return class_values[2]\n\n\nevaluations[\"class\"] = evaluations.apply(\n    lambda row: determine_class(row[flow_ids[0]], row[flow_ids[1]]), axis=1\n)\n\n# does the plotting and formatting\nfig_scatter, ax_scatter = plt.subplots()\nfor class_val in class_values:\n    df_class = evaluations[evaluations[\"class\"] == class_val]\n    plt.scatter(df_class[meta_features[0]], df_class[meta_features[1]], label=class_val)\nax_scatter.set_title(classifier_family)\nax_scatter.set_xlabel(meta_features[0])\nax_scatter.set_ylabel(meta_features[1])\nax_scatter.legend()\nax_scatter.set_xscale(\"log\")\nax_scatter.set_yscale(\"log\")\nplt.show()\n</pre> def determine_class(val_lin, val_nonlin):     if val_lin &lt; val_nonlin:         return class_values[0]     elif val_nonlin &lt; val_lin:         return class_values[1]     else:         return class_values[2]   evaluations[\"class\"] = evaluations.apply(     lambda row: determine_class(row[flow_ids[0]], row[flow_ids[1]]), axis=1 )  # does the plotting and formatting fig_scatter, ax_scatter = plt.subplots() for class_val in class_values:     df_class = evaluations[evaluations[\"class\"] == class_val]     plt.scatter(df_class[meta_features[0]], df_class[meta_features[1]], label=class_val) ax_scatter.set_title(classifier_family) ax_scatter.set_xlabel(meta_features[0]) ax_scatter.set_ylabel(meta_features[1]) ax_scatter.legend() ax_scatter.set_xscale(\"log\") ax_scatter.set_yscale(\"log\") plt.show() <p>makes a scatter plot where each data point represents the performance of the two algorithms on various axis (not in the paper)</p> In\u00a0[\u00a0]: Copied! <pre>fig_diagplot, ax_diagplot = plt.subplots()\nax_diagplot.grid(linestyle=\"--\")\nax_diagplot.plot([0, 1], ls=\"-\", color=\"black\")\nax_diagplot.plot([0.2, 1.2], ls=\"--\", color=\"black\")\nax_diagplot.plot([-0.2, 0.8], ls=\"--\", color=\"black\")\nax_diagplot.scatter(evaluations[flow_ids[0]], evaluations[flow_ids[1]])\nax_diagplot.set_xlabel(measure)\nax_diagplot.set_ylabel(measure)\nplt.show()\n# License: BSD 3-Clause\n</pre> fig_diagplot, ax_diagplot = plt.subplots() ax_diagplot.grid(linestyle=\"--\") ax_diagplot.plot([0, 1], ls=\"-\", color=\"black\") ax_diagplot.plot([0.2, 1.2], ls=\"--\", color=\"black\") ax_diagplot.plot([-0.2, 0.8], ls=\"--\", color=\"black\") ax_diagplot.scatter(evaluations[flow_ids[0]], evaluations[flow_ids[1]]) ax_diagplot.set_xlabel(measure) ax_diagplot.set_ylabel(measure) plt.show() # License: BSD 3-Clause"},{"location":"examples/40_paper/2018_ida_strang_example/#strang-et-al-2018","title":"Strang et al. (2018)\u00b6","text":"<p>A tutorial on how to reproduce the analysis conducted for Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML.</p>"},{"location":"examples/40_paper/2018_ida_strang_example/#publication","title":"Publication\u00b6","text":"<p>| Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML | Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank Hutter | In Advances in Intelligent Data Analysis XVII 17th International Symposium, 2018 | Available at https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25</p>"},{"location":"examples/40_paper/2018_kdd_rijn_example/","title":"van Rijn and Hutter (2018)","text":"<p>With the advent of automated machine learning, automated hyperparameter optimization methods are by now routinely used in data mining. However, this progress is not yet matched by equal progress on automatic analyses that yield information beyond performance-optimizing hyperparameter settings. In this example, we aim to answer the following two questions: Given an algorithm, what are generally its most important hyperparameters?</p> <p>This work is carried out on the OpenML-100 benchmark suite, which can be obtained by <code>openml.study.get_suite('OpenML100')</code>. In this example, we conduct the experiment on the Support Vector Machine (<code>flow_id=7707</code>) with specific kernel (we will perform a post-process filter operation for this). We should set some other experimental parameters (number of results per task, evaluation measure and the number of trees of the internal functional Anova) before the fun can begin.</p> <p>Note that we simplify the example in several ways:</p> <ol> <li>We only consider numerical hyperparameters</li> <li>We consider all hyperparameters that are numerical (in reality, some hyperparameters might be inactive (e.g., <code>degree</code>) or irrelevant (e.g., <code>random_state</code>)</li> <li>We assume all hyperparameters to be on uniform scale</li> </ol> <p>Any difference in conclusion between the actual paper and the presented results is most likely due to one of these simplifications. For example, the hyperparameter C looks rather insignificant, whereas it is quite important when it is put on a log-scale. All these simplifications can be addressed by defining a ConfigSpace. For a more elaborated example that uses this, please see: https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401</p> In\u00a0[\u00a0]: Copied! <pre>suite = openml.study.get_suite(\"OpenML100\")\nflow_id = 7707\nparameter_filters = {\"sklearn.svm.classes.SVC(17)_kernel\": \"sigmoid\"}\nevaluation_measure = \"predictive_accuracy\"\nlimit_per_task = 500\nlimit_nr_tasks = 15\nn_trees = 16\n\nfanova_results = []\n# we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the\n# communication with OpenML, and for iterated experimenting it is better to cache the results in a local file.\nfor idx, task_id in enumerate(suite.tasks):\n    if limit_nr_tasks is not None and idx &gt;= limit_nr_tasks:\n        continue\n    print(\n        \"Starting with task %d (%d/%d)\"\n        % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks)\n    )\n    # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop)\n    evals = openml.evaluations.list_evaluations_setups(\n        evaluation_measure,\n        flows=[flow_id],\n        tasks=[task_id],\n        size=limit_per_task,\n        output_format=\"dataframe\",\n    )\n\n    performance_column = \"value\"\n    # make a DataFrame consisting of all hyperparameters (which is a dict in setup['parameters']) and the performance\n    # value (in setup['value']). The following line looks a bit complicated, but combines 2 tasks: a) combine\n    # hyperparameters and performance data in a single dict, b) cast hyperparameter values to the appropriate format\n    # Note that the ``json.loads(...)`` requires the content to be in JSON format, which is only the case for\n    # scikit-learn setups (and even there some legacy setups might violate this requirement). It will work for the\n    # setups that belong to the flows embedded in this example though.\n    try:\n        setups_evals = pd.DataFrame(\n            [\n                dict(\n                    **{name: json.loads(value) for name, value in setup[\"parameters\"].items()},\n                    **{performance_column: setup[performance_column]}\n                )\n                for _, setup in evals.iterrows()\n            ]\n        )\n    except json.decoder.JSONDecodeError as e:\n        print(\"Task %d error: %s\" % (task_id, e))\n        continue\n    # apply our filters, to have only the setups that comply to the hyperparameters we want\n    for filter_key, filter_value in parameter_filters.items():\n        setups_evals = setups_evals[setups_evals[filter_key] == filter_value]\n    # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters,\n    # the fanova library needs to be informed by using a configspace object.\n    setups_evals = setups_evals.select_dtypes(include=[\"int64\", \"float64\"])\n    # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``,\n    # ``verbose``.\n    setups_evals = setups_evals[\n        [\n            c\n            for c in list(setups_evals)\n            if len(setups_evals[c].unique()) &gt; 1 or c == performance_column\n        ]\n    ]\n    # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g.,\n    # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out:\n\n    # determine x values to pass to fanova library\n    parameter_names = [\n        pname for pname in setups_evals.columns.to_numpy() if pname != performance_column\n    ]\n    evaluator = fanova.fanova.fANOVA(\n        X=setups_evals[parameter_names].to_numpy(),\n        Y=setups_evals[performance_column].to_numpy(),\n        n_trees=n_trees,\n    )\n    for idx, pname in enumerate(parameter_names):\n        try:\n            fanova_results.append(\n                {\n                    \"hyperparameter\": pname.split(\".\")[-1],\n                    \"fanova\": evaluator.quantify_importance([idx])[(idx,)][\"individual importance\"],\n                }\n            )\n        except RuntimeError as e:\n            # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant\n            # for all configurations (there is no variance). We will skip these tasks (like the authors did in the\n            # paper).\n            print(\"Task %d error: %s\" % (task_id, e))\n            continue\n\n# transform ``fanova_results`` from a list of dicts into a DataFrame\nfanova_results = pd.DataFrame(fanova_results)\n</pre> suite = openml.study.get_suite(\"OpenML100\") flow_id = 7707 parameter_filters = {\"sklearn.svm.classes.SVC(17)_kernel\": \"sigmoid\"} evaluation_measure = \"predictive_accuracy\" limit_per_task = 500 limit_nr_tasks = 15 n_trees = 16  fanova_results = [] # we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the # communication with OpenML, and for iterated experimenting it is better to cache the results in a local file. for idx, task_id in enumerate(suite.tasks):     if limit_nr_tasks is not None and idx &gt;= limit_nr_tasks:         continue     print(         \"Starting with task %d (%d/%d)\"         % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks)     )     # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop)     evals = openml.evaluations.list_evaluations_setups(         evaluation_measure,         flows=[flow_id],         tasks=[task_id],         size=limit_per_task,         output_format=\"dataframe\",     )      performance_column = \"value\"     # make a DataFrame consisting of all hyperparameters (which is a dict in setup['parameters']) and the performance     # value (in setup['value']). The following line looks a bit complicated, but combines 2 tasks: a) combine     # hyperparameters and performance data in a single dict, b) cast hyperparameter values to the appropriate format     # Note that the ``json.loads(...)`` requires the content to be in JSON format, which is only the case for     # scikit-learn setups (and even there some legacy setups might violate this requirement). It will work for the     # setups that belong to the flows embedded in this example though.     try:         setups_evals = pd.DataFrame(             [                 dict(                     **{name: json.loads(value) for name, value in setup[\"parameters\"].items()},                     **{performance_column: setup[performance_column]}                 )                 for _, setup in evals.iterrows()             ]         )     except json.decoder.JSONDecodeError as e:         print(\"Task %d error: %s\" % (task_id, e))         continue     # apply our filters, to have only the setups that comply to the hyperparameters we want     for filter_key, filter_value in parameter_filters.items():         setups_evals = setups_evals[setups_evals[filter_key] == filter_value]     # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters,     # the fanova library needs to be informed by using a configspace object.     setups_evals = setups_evals.select_dtypes(include=[\"int64\", \"float64\"])     # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``,     # ``verbose``.     setups_evals = setups_evals[         [             c             for c in list(setups_evals)             if len(setups_evals[c].unique()) &gt; 1 or c == performance_column         ]     ]     # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g.,     # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out:      # determine x values to pass to fanova library     parameter_names = [         pname for pname in setups_evals.columns.to_numpy() if pname != performance_column     ]     evaluator = fanova.fanova.fANOVA(         X=setups_evals[parameter_names].to_numpy(),         Y=setups_evals[performance_column].to_numpy(),         n_trees=n_trees,     )     for idx, pname in enumerate(parameter_names):         try:             fanova_results.append(                 {                     \"hyperparameter\": pname.split(\".\")[-1],                     \"fanova\": evaluator.quantify_importance([idx])[(idx,)][\"individual importance\"],                 }             )         except RuntimeError as e:             # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant             # for all configurations (there is no variance). We will skip these tasks (like the authors did in the             # paper).             print(\"Task %d error: %s\" % (task_id, e))             continue  # transform ``fanova_results`` from a list of dicts into a DataFrame fanova_results = pd.DataFrame(fanova_results) <p>make the boxplot of the variance contribution. Obviously, we can also use this data to make the Nemenyi plot, but this relies on the rather complex <code>Orange</code> dependency (<code>pip install Orange3</code>). For the complete example, the reader is referred to the more elaborate script (referred to earlier)</p> In\u00a0[\u00a0]: Copied! <pre>fig, ax = plt.subplots()\nsns.boxplot(x=\"hyperparameter\", y=\"fanova\", data=fanova_results, ax=ax)\nax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\nax.set_ylabel(\"Variance Contribution\")\nax.set_xlabel(None)\nplt.tight_layout()\nplt.show()\n# License: BSD 3-Clause\n</pre> fig, ax = plt.subplots() sns.boxplot(x=\"hyperparameter\", y=\"fanova\", data=fanova_results, ax=ax) ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\") ax.set_ylabel(\"Variance Contribution\") ax.set_xlabel(None) plt.tight_layout() plt.show() # License: BSD 3-Clause"},{"location":"examples/40_paper/2018_kdd_rijn_example/#van-rijn-and-hutter-2018","title":"van Rijn and Hutter (2018)\u00b6","text":"<p>A tutorial on how to reproduce the paper Hyperparameter Importance Across Datasets.</p> <p>This is a Unix-only tutorial, as the requirements can not be satisfied on a Windows machine (Untested on other systems).</p>"},{"location":"examples/40_paper/2018_kdd_rijn_example/#publication","title":"Publication\u00b6","text":"<p>| Hyperparameter importance across datasets | Jan N. van Rijn and Frank Hutter | In Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery &amp; Data Mining, 2018 | Available at https://dl.acm.org/doi/10.1145/3219819.3220058</p> <p>import sys</p> <p>if sys.platform == \"win32\":  # noqa print( \"The pyrfr library (requirement of fanova) can currently not be installed on Windows systems\" ) exit()</p> <p>import json import fanova import matplotlib.pyplot as plt import pandas as pd import seaborn as sns</p> <p>import openml</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/","title":"Perrone et al. (2018)","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.ensemble import RandomForestRegressor\n\nflow_type = \"svm\"  # this example will use the smaller svm flow evaluations\n</pre> import openml import numpy as np import pandas as pd from matplotlib import pyplot as plt from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer from sklearn.compose import ColumnTransformer from sklearn.metrics import mean_squared_error from sklearn.preprocessing import OneHotEncoder from sklearn.ensemble import RandomForestRegressor  flow_type = \"svm\"  # this example will use the smaller svm flow evaluations <p>The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into a tabular format that can be used to build models.</p> In\u00a0[\u00a0]: Copied! <pre>def fetch_evaluations(run_full=False, flow_type=\"svm\", metric=\"area_under_roc_curve\"):\n    \"\"\"\n    Fetch a list of evaluations based on the flows and tasks used in the experiments.\n\n    Parameters\n    ----------\n    run_full : boolean\n        If True, use the full list of tasks used in the paper\n        If False, use 5 tasks with the smallest number of evaluations available\n    flow_type : str, {'svm', 'xgboost'}\n        To select whether svm or xgboost experiments are to be run\n    metric : str\n        The evaluation measure that is passed to openml.evaluations.list_evaluations\n\n    Returns\n    -------\n    eval_df : dataframe\n    task_ids : list\n    flow_id : int\n    \"\"\"\n    # Collecting task IDs as used by the experiments from the paper\n    # fmt: off\n    if flow_type == \"svm\" and run_full:\n        task_ids = [\n            10101, 145878, 146064, 14951, 34537, 3485, 3492, 3493, 3494,\n            37, 3889, 3891, 3899, 3902, 3903, 3913, 3918, 3950, 9889,\n            9914, 9946, 9952, 9967, 9971, 9976, 9978, 9980, 9983,\n        ]\n    elif flow_type == \"svm\" and not run_full:\n        task_ids = [9983, 3485, 3902, 3903, 145878]\n    elif flow_type == \"xgboost\" and run_full:\n        task_ids = [\n            10093, 10101, 125923, 145847, 145857, 145862, 145872, 145878,\n            145953, 145972, 145976, 145979, 146064, 14951, 31, 3485,\n            3492, 3493, 37, 3896, 3903, 3913, 3917, 3918, 3, 49, 9914,\n            9946, 9952, 9967,\n        ]\n    else:  # flow_type == 'xgboost' and not run_full:\n        task_ids = [3903, 37, 3485, 49, 3913]\n    # fmt: on\n\n    # Fetching the relevant flow\n    flow_id = 5891 if flow_type == \"svm\" else 6767\n\n    # Fetching evaluations\n    eval_df = openml.evaluations.list_evaluations_setups(\n        function=metric,\n        tasks=task_ids,\n        flows=[flow_id],\n        uploaders=[2702],\n        output_format=\"dataframe\",\n        parameters_in_separate_columns=True,\n    )\n    return eval_df, task_ids, flow_id\n\n\ndef create_table_from_evaluations(\n    eval_df, flow_type=\"svm\", run_count=np.iinfo(np.int64).max, task_ids=None\n):\n    \"\"\"\n    Create a tabular data with its ground truth from a dataframe of evaluations.\n    Optionally, can filter out records based on task ids.\n\n    Parameters\n    ----------\n    eval_df : dataframe\n        Containing list of runs as obtained from list_evaluations()\n    flow_type : str, {'svm', 'xgboost'}\n        To select whether svm or xgboost experiments are to be run\n    run_count : int\n        Maximum size of the table created, or number of runs included in the table\n    task_ids : list, (optional)\n        List of integers specifying the tasks to be retained from the evaluations dataframe\n\n    Returns\n    -------\n    eval_table : dataframe\n    values : list\n    \"\"\"\n    if task_ids is not None:\n        eval_df = eval_df[eval_df[\"task_id\"].isin(task_ids)]\n    if flow_type == \"svm\":\n        colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]\n    else:\n        colnames = [\n            \"alpha\",\n            \"booster\",\n            \"colsample_bylevel\",\n            \"colsample_bytree\",\n            \"eta\",\n            \"lambda\",\n            \"max_depth\",\n            \"min_child_weight\",\n            \"nrounds\",\n            \"subsample\",\n        ]\n    eval_df = eval_df.sample(frac=1)  # shuffling rows\n    eval_df = eval_df.iloc[:run_count, :]\n    eval_df.columns = [column.split(\"_\")[-1] for column in eval_df.columns]\n    eval_table = eval_df.loc[:, colnames]\n    value = eval_df.loc[:, \"value\"]\n    return eval_table, value\n\n\ndef list_categorical_attributes(flow_type=\"svm\"):\n    if flow_type == \"svm\":\n        return [\"kernel\"]\n    return [\"booster\"]\n</pre> def fetch_evaluations(run_full=False, flow_type=\"svm\", metric=\"area_under_roc_curve\"):     \"\"\"     Fetch a list of evaluations based on the flows and tasks used in the experiments.      Parameters     ----------     run_full : boolean         If True, use the full list of tasks used in the paper         If False, use 5 tasks with the smallest number of evaluations available     flow_type : str, {'svm', 'xgboost'}         To select whether svm or xgboost experiments are to be run     metric : str         The evaluation measure that is passed to openml.evaluations.list_evaluations      Returns     -------     eval_df : dataframe     task_ids : list     flow_id : int     \"\"\"     # Collecting task IDs as used by the experiments from the paper     # fmt: off     if flow_type == \"svm\" and run_full:         task_ids = [             10101, 145878, 146064, 14951, 34537, 3485, 3492, 3493, 3494,             37, 3889, 3891, 3899, 3902, 3903, 3913, 3918, 3950, 9889,             9914, 9946, 9952, 9967, 9971, 9976, 9978, 9980, 9983,         ]     elif flow_type == \"svm\" and not run_full:         task_ids = [9983, 3485, 3902, 3903, 145878]     elif flow_type == \"xgboost\" and run_full:         task_ids = [             10093, 10101, 125923, 145847, 145857, 145862, 145872, 145878,             145953, 145972, 145976, 145979, 146064, 14951, 31, 3485,             3492, 3493, 37, 3896, 3903, 3913, 3917, 3918, 3, 49, 9914,             9946, 9952, 9967,         ]     else:  # flow_type == 'xgboost' and not run_full:         task_ids = [3903, 37, 3485, 49, 3913]     # fmt: on      # Fetching the relevant flow     flow_id = 5891 if flow_type == \"svm\" else 6767      # Fetching evaluations     eval_df = openml.evaluations.list_evaluations_setups(         function=metric,         tasks=task_ids,         flows=[flow_id],         uploaders=[2702],         output_format=\"dataframe\",         parameters_in_separate_columns=True,     )     return eval_df, task_ids, flow_id   def create_table_from_evaluations(     eval_df, flow_type=\"svm\", run_count=np.iinfo(np.int64).max, task_ids=None ):     \"\"\"     Create a tabular data with its ground truth from a dataframe of evaluations.     Optionally, can filter out records based on task ids.      Parameters     ----------     eval_df : dataframe         Containing list of runs as obtained from list_evaluations()     flow_type : str, {'svm', 'xgboost'}         To select whether svm or xgboost experiments are to be run     run_count : int         Maximum size of the table created, or number of runs included in the table     task_ids : list, (optional)         List of integers specifying the tasks to be retained from the evaluations dataframe      Returns     -------     eval_table : dataframe     values : list     \"\"\"     if task_ids is not None:         eval_df = eval_df[eval_df[\"task_id\"].isin(task_ids)]     if flow_type == \"svm\":         colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]     else:         colnames = [             \"alpha\",             \"booster\",             \"colsample_bylevel\",             \"colsample_bytree\",             \"eta\",             \"lambda\",             \"max_depth\",             \"min_child_weight\",             \"nrounds\",             \"subsample\",         ]     eval_df = eval_df.sample(frac=1)  # shuffling rows     eval_df = eval_df.iloc[:run_count, :]     eval_df.columns = [column.split(\"_\")[-1] for column in eval_df.columns]     eval_table = eval_df.loc[:, colnames]     value = eval_df.loc[:, \"value\"]     return eval_table, value   def list_categorical_attributes(flow_type=\"svm\"):     if flow_type == \"svm\":         return [\"kernel\"]     return [\"booster\"] <p>Fetching the data from OpenML</p> <p>Now, we read all the tasks and evaluations for them and collate into a table. Here, we are reading all the tasks and evaluations for the SVM flow and pre-processing all retrieved evaluations.</p> In\u00a0[\u00a0]: Copied! <pre>eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type)\nX, y = create_table_from_evaluations(eval_df, flow_type=flow_type)\nprint(X.head())\nprint(\"Y : \", y[:5])\n</pre> eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type) X, y = create_table_from_evaluations(eval_df, flow_type=flow_type) print(X.head()) print(\"Y : \", y[:5]) In\u00a0[\u00a0]: Copied! <pre># Separating data into categorical and non-categorical (numeric for this example) columns\ncat_cols = list_categorical_attributes(flow_type=flow_type)\nnum_cols = list(set(X.columns) - set(cat_cols))\n\n# Missing value imputers for numeric columns\nnum_imputer = SimpleImputer(missing_values=np.nan, strategy=\"constant\", fill_value=-1)\n\n# Creating the one-hot encoder for numerical representation of categorical columns\nenc = OneHotEncoder(handle_unknown=\"ignore\")\n\n# Combining column transformers\nct = ColumnTransformer([(\"cat\", enc, cat_cols), (\"num\", num_imputer, num_cols)])\n\n# Creating the full pipeline with the surrogate model\nclf = RandomForestRegressor(n_estimators=50)\nmodel = Pipeline(steps=[(\"preprocess\", ct), (\"surrogate\", clf)])\n</pre> # Separating data into categorical and non-categorical (numeric for this example) columns cat_cols = list_categorical_attributes(flow_type=flow_type) num_cols = list(set(X.columns) - set(cat_cols))  # Missing value imputers for numeric columns num_imputer = SimpleImputer(missing_values=np.nan, strategy=\"constant\", fill_value=-1)  # Creating the one-hot encoder for numerical representation of categorical columns enc = OneHotEncoder(handle_unknown=\"ignore\")  # Combining column transformers ct = ColumnTransformer([(\"cat\", enc, cat_cols), (\"num\", num_imputer, num_cols)])  # Creating the full pipeline with the surrogate model clf = RandomForestRegressor(n_estimators=50) model = Pipeline(steps=[(\"preprocess\", ct), (\"surrogate\", clf)]) In\u00a0[\u00a0]: Copied! <pre># Selecting a task for the surrogate\ntask_id = task_ids[-1]\nprint(\"Task ID : \", task_id)\nX, y = create_table_from_evaluations(eval_df, task_ids=[task_id], flow_type=\"svm\")\n\nmodel.fit(X, y)\ny_pred = model.predict(X)\n\nprint(\"Training RMSE : {:.5}\".format(mean_squared_error(y, y_pred)))\n</pre> # Selecting a task for the surrogate task_id = task_ids[-1] print(\"Task ID : \", task_id) X, y = create_table_from_evaluations(eval_df, task_ids=[task_id], flow_type=\"svm\")  model.fit(X, y) y_pred = model.predict(X)  print(\"Training RMSE : {:.5}\".format(mean_squared_error(y, y_pred))) In\u00a0[\u00a0]: Copied! <pre># Sampling random configurations\ndef random_sample_configurations(num_samples=100):\n    colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]\n    ranges = [\n        (0.000986, 998.492437),\n        (2.0, 5.0),\n        (0.000988, 913.373845),\n        ([\"linear\", \"polynomial\", \"radial\", \"sigmoid\"]),\n    ]\n    X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames)\n    for i in range(len(colnames)):\n        if len(ranges[i]) == 2:\n            col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], size=num_samples)\n        else:\n            col_val = np.random.choice(ranges[i], size=num_samples)\n        X.iloc[:, i] = col_val\n    return X\n\n\nconfigs = random_sample_configurations(num_samples=1000)\nprint(configs)\n</pre> # Sampling random configurations def random_sample_configurations(num_samples=100):     colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]     ranges = [         (0.000986, 998.492437),         (2.0, 5.0),         (0.000988, 913.373845),         ([\"linear\", \"polynomial\", \"radial\", \"sigmoid\"]),     ]     X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames)     for i in range(len(colnames)):         if len(ranges[i]) == 2:             col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], size=num_samples)         else:             col_val = np.random.choice(ranges[i], size=num_samples)         X.iloc[:, i] = col_val     return X   configs = random_sample_configurations(num_samples=1000) print(configs) In\u00a0[\u00a0]: Copied! <pre>preds = model.predict(configs)\n\n# tracking the maximum AUC obtained over the functions evaluations\npreds = np.maximum.accumulate(preds)\n# computing regret (1 - predicted_auc)\nregret = 1 - preds\n\n# plotting the regret curve\nplt.plot(regret)\nplt.title(\"AUC regret for Random Search on surrogate\")\nplt.xlabel(\"Numbe of function evaluations\")\nplt.ylabel(\"Regret\")\n# License: BSD 3-Clause\n</pre> preds = model.predict(configs)  # tracking the maximum AUC obtained over the functions evaluations preds = np.maximum.accumulate(preds) # computing regret (1 - predicted_auc) regret = 1 - preds  # plotting the regret curve plt.plot(regret) plt.title(\"AUC regret for Random Search on surrogate\") plt.xlabel(\"Numbe of function evaluations\") plt.ylabel(\"Regret\") # License: BSD 3-Clause"},{"location":"examples/40_paper/2018_neurips_perrone_example/#perrone-et-al-2018","title":"Perrone et al. (2018)\u00b6","text":"<p>A tutorial on how to build a surrogate model based on OpenML data as done for Scalable Hyperparameter Transfer Learning by Perrone et al..</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#publication","title":"Publication\u00b6","text":"<p>| Scalable Hyperparameter Transfer Learning | Valerio Perrone and Rodolphe Jenatton and Matthias Seeger and Cedric Archambeau | In Advances in Neural Information Processing Systems 31, 2018 | Available at https://papers.nips.cc/paper/7917-scalable-hyperparameter-transfer-learning.pdf</p> <p>This example demonstrates how OpenML runs can be used to construct a surrogate model.</p> <p>In the following section, we shall do the following:</p> <ul> <li>Retrieve tasks and flows as used in the experiments by Perrone et al. (2018).</li> <li>Build a tabular data by fetching the evaluations uploaded to OpenML.</li> <li>Impute missing values and handle categorical data before building a Random Forest model that maps hyperparameter values to the area under curve score.</li> </ul>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#creating-pre-processing-and-modelling-pipelines","title":"Creating pre-processing and modelling pipelines\u00b6","text":"<p>The two primary tasks are to impute the missing values, that is, account for the hyperparameters that are not available with the runs from OpenML. And secondly, to handle categorical variables using One-hot encoding prior to modelling.</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#building-a-surrogate-model-on-a-tasks-evaluation","title":"Building a surrogate model on a task's evaluation\u00b6","text":"<p>The same set of functions can be used for a single task to retrieve a singular table which can be used for the surrogate model construction. We shall use the SVM flow here to keep execution time simple and quick.</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#evaluating-the-surrogate-model","title":"Evaluating the surrogate model\u00b6","text":"<p>The surrogate model built from a task's evaluations fetched from OpenML will be put into trivial action here, where we shall randomly sample configurations and observe the trajectory of the area under curve (auc) we can obtain from the surrogate we've built.</p> <p>NOTE: This section is written exclusively for the SVM flow</p>"},{"location":"help/","title":"Questions","text":"<p>If you have questions, you can contact us in various ways. Here are the best ways to reach out depending on your questions.</p>"},{"location":"help/#general-questions","title":"General questions","text":"<p>For all non-technical questions, such as how to get involved, collaboration ideas, and interesting proposals, please should the communication channel of your choice. We believe in making the world better, together, so we're always happy to hear from you </p> <ul> <li> <p> Contact us via email</p> <p>Old school. We love it :). This email reaches all the main OpenML developers.</p> <p> Write an email</p> </li> <li> <p> Chat with us on Slack</p> <p>Get involved in current discussions or start your own.</p> <p> Join us on Slack</p> </li> <li> <p> Social media</p> <p>You can reach us on X (formerly Twitter)</p> <p> Post something now</p> </li> </ul>"},{"location":"help/#technical-questions","title":"Technical questions","text":"<p>For technical questions, the best way is to open an issue on GitHub. We have several issue trackers which are closely monitored by the people who can best answer your questions. We are happy to help, so don't be shy </p>"},{"location":"help/#openml-client-libraries","title":"OpenML client libraries","text":"<ul> <li> <p> Using OpenML in Python</p> <p>For all questions on using OpenML via the Python API</p> <p> Post your question</p> </li> <li> <p> Using OpenML in R</p> <p>For all questions on using OpenML via the R API (via mlr3)</p> <p> Post your question</p> </li> <li> <p> Using OpenML in Julia</p> <p>For all questions on using OpenML via the Julia API</p> <p> Post your question</p> </li> <li> <p> Using OpenML in Java</p> <p>For all questions on using OpenML via the Java API</p> <p> Post your question</p> </li> </ul>"},{"location":"help/#openml-platform","title":"OpenML platform","text":"<ul> <li> <p> OpenML Server</p> <p>For all questions on the OpenML server and REST API</p> <p> Version 1 of the API (PHP)  Version 2 of the API (Python)</p> </li> <li> <p> OpenML Website</p> <p>For all questions about the OpenML website</p> <p> Post your question</p> </li> <li> <p> OpenML deployment</p> <p>For all questions on deploying OpenML locally or how OpenML runs in production</p> <p> Post your question</p> </li> <li> <p> OpenML documentation</p> <p>For all questions and suggestions for these documentation pages</p> <p> Post your question</p> </li> </ul>"},{"location":"help/#openml-content","title":"OpenML content","text":"<ul> <li> <p> Datasets</p> <p>For all questions about OpenML datasets</p> <p> Post your question</p> </li> <li> <p> Croissant</p> <p>For all questions about Croissant, the metadata standard we use for describing machine learning datasets</p> <p> Post your question</p> </li> </ul>"},{"location":"intro/Governance/","title":"Governance","text":"<p>The purpose of this document is to formalize the governance process used by the OpenML project (the OpenML GitHub organization which contains all code and projects related to OpenML.org), to clarify how decisions are made and how the various elements of our community interact. This document establishes a decision-making structure that takes into account feedback from all members of the community and strives to find consensus, while avoiding any deadlocks.</p> <p>The OpenML project is an independent open source project that is legally represented by the Open Machine Learning Foundation. The Open Machine Learning Foundation is a not-for-profit organization supporting, but not controlling, the OpenML project. The Foundation is open to engage with universities, companies, or anyone sharing the same goals. The OpenML project has a separate governance model described in this document.</p> <p>This is a meritocratic, consensus-based community project. Anyone with an interest in the project can join the community, contribute to the project design, and participate in the decision making process. This document describes how that participation takes place and how to set about earning merit within the project community.</p>"},{"location":"intro/Governance/#roles-and-responsibilities","title":"Roles And Responsibilities","text":""},{"location":"intro/Governance/#contributors","title":"Contributors","text":"<p>Contributors are community members who contribute in concrete ways to the project. Anyone can become a contributor, and contributions can take many forms \u2013 not only code \u2013 as detailed in the contributors guide. Contributors need to create pull requests to contribute to the code or documentation.</p>"},{"location":"intro/Governance/#core-contributors","title":"Core contributors","text":"<p>Core contributors are community members who have shown that they are dedicated to the continued development of the project through ongoing engagement with the community. They have shown they can be trusted to maintain OpenML with care. Being a core contributor allows contributors to more easily carry on with their project related activities by giving them write access to the project\u2019s repository (abiding by the decision making process described below, e.g. merging pull requests that obey the decision making procedure described below) and is represented as being an organization member on the OpenML GitHub organization. Core contributors are expected to review code contributions, can merge approved pull requests, can cast votes for and against merging a pull-request, and can be involved in deciding major changes to the API.</p> <p>New core contributors can be nominated by any existing core contributors. Once they have been nominated, there will be a vote in the private OpenML core email list by the current core contributors. While it is expected that most votes will be unanimous, a two-thirds majority of the cast votes is enough. The vote needs to be open for at least 1 week.</p> <p>Core contributors that have not contributed to the project (commits or GitHub comments) in the past 12 months will become emeritus core contributors and recant their commit and voting rights until they become active again. The list of core contributors, active and emeritus (with dates at which they became active) is public on the OpenML website.</p>"},{"location":"intro/Governance/#steering-committee","title":"Steering Committee","text":"<p>The Steering Committee (SC) members are core contributors who have additional responsibilities to ensure the smooth running of the project. SC members are expected to participate in strategic planning, join monthly meetings, and approve changes to the governance model. The purpose of the SC is to ensure a smooth progress from the big-picture perspective. Indeed, changes that impact the full project require a synthetic analysis and a consensus that is both explicit and informed. In cases that the core contributor community (which includes the SC members) fails to reach such a consensus in the required time frame, the SC is the entity to resolve the issue.</p> <p>The SC consists of community representatives and partner representatives. Community representatives of the SC are nominated by a core contributor. A nomination will result in a discussion that cannot take more than a month and then a vote by the core contributors which will stay open for a week. SC membership votes are subject to a two-third majority of all cast votes as well as a simple majority approval of all the current SC members.</p> <p>Partner institutions who enter a collaboration agreement or sponsorship agreement with the OpenML Foundation can nominate a representative on the Steering Committee, if so agreed in the agreement. Such a collaboration should in principle include one full-time developer to work on OpenML (in cash or in kind) for the duration of the agreement. New partner representatives have to be confirmed by the SC following the same voting rules above.</p> <p>The OpenML community must have at least equal footing in the steering committee. Additional SC members may be nominated to ensure this, following the membership voting rules described above.</p> <p>When decisions are escalated to the steering committee (see the decision making process below), and no consensus can be found within a month, the SC can meet and decide by consensus or with a simple majority of all cast votes.</p> <p>SC members who do not actively engage with the SC duties are expected to resign.</p> <p>The initial Steering Committee of OpenML consists of Bernd Bischl, Giuseppe Casalicchio, Matthias Feurer, Heidi Seibold, Jan van Rijn, and Joaquin Vanschoren. They all represent the OpenML community.</p>"},{"location":"intro/Governance/#decision-making-process","title":"Decision Making Process","text":"<p>Decisions about the future of the project are made through discussion with all members of the community. All non-sensitive project management discussion takes place on the project contributors\u2019 mailing list and the issue trackers of the sub-projects. Occasionally, sensitive discussion occurs on the private core developer email list (see below). This includes voting on core/SC membership or discussion of internal disputes. All discussions must follow the OpenML honor code.</p> <p>OpenML uses a \u201cconsensus seeking\u201d process for making decisions. The group tries to find a resolution that has no open objections among core contributors. At any point during the discussion, any core contributors can call for a vote, which will conclude one month from the call for the vote, or when two thirds of all votes are in favor.</p> <p>If no option can gather two thirds of the votes cast (ignoring abstentions), the decision is escalated to the SC, which in turn will use consensus seeking with the fallback option of a simple majority vote if no consensus can be found within a month. This is what we hereafter may refer to as \u201cthe decision making process\u201d. It applies to all core OpenML repositories.</p> <p>Decisions (in addition to adding core contributors and SC membership as above) are made according to the following rules:</p> <p>Normal changes:  </p> <ul> <li>Minor Documentation changes, such as typo fixes, or addition / correction of a sentence: requires one approved review by a core contributor, and no objections in the comments (lazy consensus). Core contributors are expected to give \u201creasonable time\u201d to others to give their opinion on the pull request if they\u2019re not confident others would agree.  </li> <li>Non-server packages that only have one core contributor are not subject to the ruling in the bullet point above (i.e. a sole core developer can make decisions on their own).</li> </ul> <p>Major changes:  </p> <ul> <li>Major changes to the API principles and metadata schema require a concrete proposal outlined in an OpenML Request for Comments (RfC), which has to be opened for public consultation for at least 1 month. The final version has to be approved using the decision-making process outlined above (two-third of the cast vote by core contributors or simple majority if escalated to the SC). Voting is typically done as a comment in the pull request (+1, -1, or 0 to abstain).  </li> <li>RfCs must be announced and shared via the public mailing list and may link additional content (such as blog posts or google docs etc. detailing the changes).  </li> <li>Changes to the governance model use the same decision process outlined above.  </li> </ul> <p>If a veto -1 vote is cast on a lazy consensus, the proposer can appeal to the community and core contributors and the change can be approved or rejected using the decision making procedure outlined above.</p>"},{"location":"intro/Governance/#communication-channels","title":"Communication channels","text":"<p>OpenML uses the following communication channels:  </p> <ul> <li>An open contributor mailing list and the GitHub issue trackers.  </li> <li>A chat application for daily interaction with the community (currently Slack).  </li> <li>Private email lists (without archive) for the core developers (core@openml.org) and steering committee (steering@openml.org), for membership voting and sensitive discussions.  </li> <li>Biyearly Steering Committee meeting at predefined times, listed on the website, and asynchronous discussions on a discussion board. They are open to all steering committee members and core contributors, and they can all request discussion on a topic. Closed meetings for SC members only can be called in if there are sensitive discussions or other valid reasons.  </li> <li>A monthly Engineering meeting at predefined times, listed on the website. The meeting is open to all. Discussion points are put on the [project roadmap]  (https://github.com/orgs/openml/projects/2).</li> </ul>"},{"location":"intro/showcase/","title":"Research using OpenML","text":"<p>This page will have a list of interesting research papers that have used OpenML. If you have used OpenML in your research and would like to have your paper listed here, please drop a PR with the relevant information (click the  icon above).</p>"},{"location":"intro/terms/","title":"Terms","text":""},{"location":"intro/terms/#honor-code","title":"Honor Code","text":"<p>By joining OpenML, you join a special worldwide community of data scientists building on each other's results and connecting their minds as efficiently as possible. This community depends on your motivation to share data, tools and ideas, and to do so with honesty. In return, you will gain trust, visibility and reputation, igniting online collaborations and studies that otherwise may not have happened.</p> <p>By using any part of OpenML, you agree to:</p> <ul> <li>Give credit where credit is due. Cite the authors whose work you are building on, or build collaborations where appropriate.</li> <li>Give back to the community by sharing your own data as openly and as soon as possible, or by helping the community in other ways. In doing so, you gain visibility and impact (citations).</li> <li>Share data according to your best efforts. Everybody make mistakes, but we trust you to correct them as soon as possible. Remove or flag data that cannot be trusted.</li> <li>Be polite and constructive in all discussions. Criticism of methods is welcomed, but personal criticisms should be avoided.</li> <li>Respect circles of trust. OpenML allows you to collaborate in 'circles' of trusted people to share unpublished results. Be considerate in sharing data with people outside this circle.</li> <li>Do not steal the work of people who openly share it. OpenML makes it easy to find all shared data (and when it was shared), thus everybody will know if you do this.</li> </ul>"},{"location":"intro/terms/#terms-of-use","title":"Terms of Use","text":"<p>You agree that you are responsible for your own use of OpenML.org and all content submitted by you, in accordance with the Honor Code and all applicable local, state, national and international laws.</p> <p>By submitting or distributing content from OpenML.org, you affirm that you have the necessary rights, licenses, consents and/or permissions to reproduce and publish this content. You cannot upload sensitive or confidential data. You, and not the developers of OpenML.org, are solely responsible for your submissions.</p> <p>By submitting content to OpenML.org, you grant OpenML.org the right to host, transfer, display and use this content, in accordance with your sharing settings and any licences granted by you. You also grant to each user a non-exclusive license to access and use this content for their own research purposes, in accordance with any licences granted by you.</p> <p>You may maintain one user account and not let anyone else use your username and/or password. You may not impersonate other persons.</p> <p>You will not intend to damage, disable, or impair any OpenML server or interfere with any other party's use and enjoyment of the service. You may not attempt to gain unauthorized access to the Site, other accounts, computer systems or networks connected to any OpenML server. You may not obtain or attempt to obtain any materials or information not intentionally made available through OpenML.</p> <p>Strictly prohibited are content that defames, harasses or threatens others, that infringes another's intellectual property, as well as indecent or unlawful content, advertising, or intentionally inaccurate information posted with the intent of misleading others. It is also prohibited to post code containing viruses, malware, spyware or any other similar software that may damage the operation of another's computer or property.</p>"},{"location":"julia/","title":"Home","text":""},{"location":"julia/#openmljl-documentation","title":"OpenML.jl Documentation","text":"<p>This is the reference documentation of <code>OpenML.jl</code>.</p> <p>The OpenML platform provides an integration platform for carrying out and comparing machine learning solutions across a broad collection of public datasets and software platforms.</p> <p>Summary of OpenML.jl functionality:</p> <ul> <li><code>OpenML.list_tags</code><code>()</code>: for listing all dataset tags</li> <li><code>OpenML.list_datasets</code><code>(; tag=nothing, filter=nothing, output_format=...)</code>: for listing available datasets</li> <li><code>OpenML.describe_dataset</code><code>(id)</code>: to describe a particular dataset</li> <li><code>OpenML.load</code><code>(id; parser=:arff)</code>: to download a dataset</li> </ul> <p></p> <p></p>"},{"location":"julia/#installation","title":"Installation","text":"<pre><code>using Pkg\nPkg.add(\"OpenML\")\n</code></pre> <p>If running the demonstration below:</p> <pre><code>Pkg.add(\"DataFrames\") \nPkg.add(\"ScientificTypes\")\n</code></pre> <p></p> <p></p>"},{"location":"julia/#sample-usage","title":"Sample usage","text":"<pre><code>julia&gt; using OpenML # or using MLJ\n\n\njulia&gt; using DataFrames\n\n\njulia&gt; OpenML.list_tags()\n300-element Vector{Any}:\n \"study_41\"\n \"uci\"\n \"study_34\"\n \"study_37\"\n \"mythbusting_1\"\n \"OpenML-CC18\"\n \"study_99\"\n \"artificial\"\n \"BNG\"\n \"study_16\"\n \u22ee\n \"Earth Science\"\n \"Social Media\"\n \"Meteorology\"\n \"Geography\"\n \"Language\"\n \"Computational Universe\"\n \"History\"\n \"Culture\"\n \"Sociology\"\n</code></pre> <p>Listing all datasets with the \"OpenML100\" tag which also have <code>n</code> instances and <code>p</code> features, where <code>100 &lt; n &lt; 1000</code> and <code>1 &lt; p &lt; 10</code>:</p> <pre><code>julia&gt; ds = OpenML.list_datasets(\n                 tag = \"OpenML100\",\n                 filter = \"number_instances/100..1000/number_features/1..10\",\n                 output_format = DataFrame)\n12\u00d713 DataFrame\n Row \u2502 id     name                              status  MajorityClassSize  Max \u22ef\n     \u2502 Int64  String                            String  Int64?             Int \u22ef\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502    11  balance-scale                     active                288      \u22ef\n   2 \u2502    15  breast-w                          active                458\n   3 \u2502    37  diabetes                          active                500\n   4 \u2502    50  tic-tac-toe                       active                626\n   5 \u2502   333  monks-problems-1                  active                278      \u22ef\n   6 \u2502   334  monks-problems-2                  active                395\n   7 \u2502   335  monks-problems-3                  active                288\n   8 \u2502   451  irish                             active                278\n   9 \u2502   469  analcatdata_dmft                  active                155      \u22ef\n  10 \u2502   470  profb                             active                448\n  11 \u2502  1464  blood-transfusion-service-center  active                570\n  12 \u2502 40496  LED-display-domain-7digit         active                 57\n                                                               9 columns omitted\n</code></pre> <p>Describing and loading one of these datasets:</p> <pre><code>julia&gt; OpenML.describe_dataset(15)\n  Author: Dr. William H. Wolberg, University of Wisconsin Source: UCI\n  (https://archive.ics.uci.edu/ml/datasets/breast+cancer+wisconsin+(original)),\n  University of Wisconsin (http://pages.cs.wisc.edu/~olvi/uwmp/cancer.html) -\n  1995 Please cite: See below, plus UCI\n  (https://archive.ics.uci.edu/ml/citation_policy.html)\n\n  Breast Cancer Wisconsin (Original) Data Set. Features are computed from a\n  digitized image of a fine needle aspirate (FNA) of a breast mass. They\n  describe characteristics of the cell nuclei present in the image. The target\n  feature records the prognosis (malignant or benign). Original data available\n  here (ftp://ftp.cs.wisc.edu/math-prog/cpo-dataset/machine-learn/cancer/)\n\n  Current dataset was adapted to ARFF format from the UCI version. Sample code\n  ID's were removed.\n\n  ! Note that there is also a related Breast Cancer Wisconsin (Diagnosis) Data\n  Set with a different set of features, better known as wdbc\n  (https://www.openml.org/d/1510).\n\n  Relevant Papers\n  \u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\n\n  W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction\n  for breast tumor diagnosis. IS&amp;T/SPIE 1993 International Symposium on\n  Electronic Imaging: Science and Technology, volume 1905, pages 861-870, San\n  Jose, CA, 1993.\n\n  O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and\n  prognosis via linear programming. Operations Research, 43(4), pages 570-577,\n  July-August 1995.\n\n  Citation request\n  \u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\n\n  This breast cancer database was obtained from the University of Wisconsin\n  Hospitals, Madison from Dr. William H. Wolberg. If you publish results when\n  using this database, then please include this information in your\n  acknowledgments. Also, please cite one or more of:\n\n    1. O. L. Mangasarian and W. H. Wolberg: \"Cancer diagnosis via linear\n       programming\", SIAM News, Volume 23, Number 5, September 1990, pp 1\n       &amp; 18.\n\n    2. William H. Wolberg and O.L. Mangasarian: \"Multisurface method of\n       pattern separation for medical diagnosis applied to breast\n       cytology\", Proceedings of the National Academy of Sciences,\n       U.S.A., Volume 87, December 1990, pp 9193-9196.\n\n    3. O. L. Mangasarian, R. Setiono, and W.H. Wolberg: \"Pattern\n       recognition via linear programming: Theory and application to\n       medical diagnosis\", in: \"Large-scale numerical optimization\",\n       Thomas F. Coleman and Yuying Li, editors, SIAM Publications,\n       Philadelphia 1990, pp 22-30.\n\n    4. K. P. Bennett &amp; O. L. Mangasarian: \"Robust linear programming\n       discrimination of two linearly inseparable sets\", Optimization\n       Methods and Software 1, 1992, 23-34 (Gordon &amp; Breach Science\n       Publishers).\n\njulia&gt; table = OpenML.load(15)\nTables.DictColumnTable with 699 rows, 10 columns, and schema:\n :Clump_Thickness        Float64\n :Cell_Size_Uniformity   Float64\n :Cell_Shape_Uniformity  Float64\n :Marginal_Adhesion      Float64\n :Single_Epi_Cell_Size   Float64\n :Bare_Nuclei            Union{Missing, Float64}\n :Bland_Chromatin        Float64\n :Normal_Nucleoli        Float64\n :Mitoses                Float64\n :Class                  CategoricalArrays.CategoricalValue{String, UInt32}\n</code></pre> <p>Converting to a data frame:</p> <pre><code>julia&gt; df = DataFrame(table)\n699\u00d710 DataFrame\n Row \u2502 Clump_Thickness  Cell_Size_Uniformity  Cell_Shape_Uniformity  Marginal_ \u22ef\n     \u2502 Float64          Float64               Float64                Float64   \u22ef\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502             5.0                   1.0                    1.0            \u22ef\n   2 \u2502             5.0                   4.0                    4.0\n   3 \u2502             3.0                   1.0                    1.0\n   4 \u2502             6.0                   8.0                    8.0\n   5 \u2502             4.0                   1.0                    1.0            \u22ef\n   6 \u2502             8.0                  10.0                   10.0\n   7 \u2502             1.0                   1.0                    1.0\n   8 \u2502             2.0                   1.0                    2.0\n  \u22ee  \u2502        \u22ee                  \u22ee                      \u22ee                    \u22ee \u22f1\n 693 \u2502             3.0                   1.0                    1.0            \u22ef\n 694 \u2502             3.0                   1.0                    1.0\n 695 \u2502             3.0                   1.0                    1.0\n 696 \u2502             2.0                   1.0                    1.0\n 697 \u2502             5.0                  10.0                   10.0            \u22ef\n 698 \u2502             4.0                   8.0                    6.0\n 699 \u2502             4.0                   8.0                    8.0\n                                                  7 columns and 684 rows omitted\n</code></pre> <p>Inspecting it's schema:</p> <pre><code>julia&gt; using ScientificTypes\n\n\njulia&gt; schema(table)\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2502 names                 \u2502 scitypes                   \u2502 types                   \u22ef\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2502 Clump_Thickness       \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Cell_Size_Uniformity  \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Cell_Shape_Uniformity \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Marginal_Adhesion     \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Single_Epi_Cell_Size  \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Bare_Nuclei           \u2502 Union{Missing, Continuous} \u2502 Union{Missing, Float64} \u22ef\n\u2502 Bland_Chromatin       \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Normal_Nucleoli       \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Mitoses               \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Class                 \u2502 Multiclass{2}              \u2502 CategoricalValue{String \u22ef\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n                                                                1 column omitted\n</code></pre> <p></p> <p></p>"},{"location":"julia/#public-api","title":"Public API","text":""},{"location":"julia/#openmllist_tags","title":"<code>OpenML.list_tags</code>","text":"<pre><code>list_tags()\n</code></pre> <p>List all available tags.</p>"},{"location":"julia/#openmllist_datasets","title":"<code>OpenML.list_datasets</code>","text":"<pre><code>list_datasets(; tag = nothing, filters = \"\", output_format = NamedTuple)\n</code></pre> <p>Lists all active OpenML datasets, if <code>tag = nothing</code> (default). To list only datasets with a given tag, choose one of the tags in <code>list_tags()</code>. An alternative <code>output_format</code> can be chosen, e.g. <code>DataFrame</code>, if the <code>DataFrames</code> package is loaded.</p> <p>A filter is a string of <code>&lt;data quality&gt;/&lt;range&gt;</code> or <code>&lt;data quality&gt;/&lt;value&gt;</code> pairs, concatenated using <code>/</code>, such as</p> <pre><code>    filter = \"number_features/10/number_instances/500..10000\"\n</code></pre> <p>The allowed data qualities include <code>tag</code>, <code>status</code>, <code>limit</code>, <code>offset</code>, <code>data_id</code>, <code>data_name</code>, <code>data_version</code>, <code>uploader</code>, <code>number_instances</code>, <code>number_features</code>, <code>number_classes</code>, <code>number_missing_values</code>.</p> <p>For more on the format and effect of <code>filters</code> refer to the openml API.</p> <p>Examples</p> <pre><code>julia&gt; using DataFrames\n\njulia&gt; ds = OpenML.list_datasets(\n               tag = \"OpenML100\",\n               filter = \"number_instances/100..1000/number_features/1..10\",\n               output_format = DataFrame\n)\n\njulia&gt; sort!(ds, :NumberOfFeatures)\n</code></pre>"},{"location":"julia/#openmldescribe_dataset","title":"<code>OpenML.describe_dataset</code>","text":"<pre><code>describe_dataset(id)\n</code></pre> <p>Load and show the OpenML description of the data set <code>id</code>. Use <code>list_datasets</code> to browse available data sets.</p> <p>Examples</p> <pre><code>julia&gt; OpenML.describe_dataset(6)\n  Author: David J. Slate Source: UCI\n  (https://archive.ics.uci.edu/ml/datasets/Letter+Recognition) - 01-01-1991 Please cite: P.\n  W. Frey and D. J. Slate. \"Letter Recognition Using Holland-style Adaptive Classifiers\".\n  Machine Learning 6(2), 1991\n\n    1. TITLE:\n\n  Letter Image Recognition Data\n\n  The objective is to identify each of a large number of black-and-white\n  rectangular pixel displays as one of the 26 capital letters in the English\n  alphabet.  The character images were based on 20 different fonts and each\n  letter within these 20 fonts was randomly distorted to produce a file of\n  20,000 unique stimuli.  Each stimulus was converted into 16 primitive\n  numerical attributes (statistical moments and edge counts) which were then\n  scaled to fit into a range of integer values from 0 through 15.  We\n  typically train on the first 16000 items and then use the resulting model\n  to predict the letter category for the remaining 4000.  See the article\n  cited above for more details.\n</code></pre>"},{"location":"julia/#openmlload","title":"<code>OpenML.load</code>","text":"<pre><code>OpenML.load(id; maxbytes = nothing)\n</code></pre> <p>Load the OpenML dataset with specified <code>id</code>, from those listed by <code>list_datasets</code> or on the OpenML site.</p> <p>Datasets are saved as julia artifacts so that they persist locally once loaded.</p> <p>Returns a table.</p> <p>Examples</p> <pre><code>using DataFrames\ntable = OpenML.load(61)\ndf = DataFrame(table) # transform to a DataFrame\nusing ScientificTypes\ndf2 = coerce(df, autotype(df)) # corce to automatically detected scientific types\n\npeek_table = OpenML.load(61, maxbytes = 1024) # load only the first 1024 bytes of the table\n</code></pre>"},{"location":"notebooks/getting_started/","title":"Getting Started","text":"In\u00a0[2]: Copied! <pre>!pip install -q openml\n</pre> !pip install -q openml In\u00a0[3]: Copied! <pre># License: BSD 3-Clause\n\nimport openml\nfrom sklearn import neighbors\n</pre> # License: BSD 3-Clause  import openml from sklearn import neighbors In\u00a0[4]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <pre>/var/folders/0t/5d8ttqzd773fy0wq3h5db0xr0000gn/T/ipykernel_60921/256497051.py:1: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!\n  openml.config.start_using_configuration_for_example()\n</pre> <p>When using the main server instead, make sure your apikey is configured. This can be done with the following line of code (uncomment it!). Never share your apikey with others.</p> In\u00a0[5]: Copied! <pre># openml.config.apikey = 'YOURKEY'\n</pre> # openml.config.apikey = 'YOURKEY' In\u00a0[6]: Copied! <pre># Uncomment and set your OpenML cache directory\n# import os\n# openml.config.cache_directory = os.path.expanduser('YOURDIR')\n</pre> # Uncomment and set your OpenML cache directory # import os # openml.config.cache_directory = os.path.expanduser('YOURDIR') In\u00a0[7]: Copied! <pre>task = openml.tasks.get_task(403)\ndata = openml.datasets.get_dataset(task.dataset_id)\nclf = neighbors.KNeighborsClassifier(n_neighbors=5)\nrun = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n# Publish the experiment on OpenML (optional, requires an API key).\n# For this tutorial, our configuration publishes to the test server\n# as to not crowd the main server with runs created by examples.\nmyrun = run.publish()\nprint(f\"kNN on {data.name}: {myrun.openml_url}\")\n</pre> task = openml.tasks.get_task(403) data = openml.datasets.get_dataset(task.dataset_id) clf = neighbors.KNeighborsClassifier(n_neighbors=5) run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False) # Publish the experiment on OpenML (optional, requires an API key). # For this tutorial, our configuration publishes to the test server # as to not crowd the main server with runs created by examples. myrun = run.publish() print(f\"kNN on {data.name}: {myrun.openml_url}\") <pre>kNN on eeg-eye-state: https://test.openml.org/r/32906\n</pre> In\u00a0[8]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n</pre> openml.config.stop_using_configuration_for_example()"},{"location":"notebooks/getting_started/#getting-started","title":"Getting Started\u00b6","text":"<p>This page will guide you through the process of getting started with OpenML. While this page is a good starting point, for more detailed information, please refer to the integrations section and the rest of the documentation.</p>"},{"location":"notebooks/getting_started/#authentication","title":"Authentication\u00b6","text":"<ul> <li>If you are using the OpenML API to download datasets, upload results, or create tasks, you will need to authenticate. You can do this by creating an account on the OpenML website and using your API key. - You can find detailed instructions on how to authenticate in the authentication section</li> </ul>"},{"location":"notebooks/getting_started/#eeg-eye-state-example","title":"EEG Eye State example\u00b6","text":"<p>Download the OpenML task for the eeg-eye-state.</p>"},{"location":"notebooks/getting_started/#caching","title":"Caching\u00b6","text":"<p>When downloading datasets, tasks, runs and flows, they will be cached to retrieve them without calling the server later. As with the API key, the cache directory can be either specified through the config file or through the API:</p> <ul> <li>Add the  line cachedir = 'MYDIR' to the config file, replacing 'MYDIR' with the path to the cache directory. By default, OpenML will use ~/.openml/cache as the cache directory.</li> <li>Run the code below, replacing 'YOURDIR' with the path to the cache directory.</li> </ul>"},{"location":"python/","title":"OpenML-Python","text":"<p>A python interface for OpenML, an online platform for open science collaboration in machine learning. It can be used to download or upload OpenML data such as datasets and machine learning experiment results.</p>"},{"location":"python/#general","title":"General","text":"<ul> <li>Documentation.</li> <li>Contribution guidelines.</li> </ul>"},{"location":"python/#citing-openml-python","title":"Citing OpenML-Python","text":"<p>If you use OpenML-Python in a scientific publication, we would appreciate a reference to the following paper:</p> <p>Matthias Feurer, Jan N. van Rijn, Arlind Kadra, Pieter Gijsbers, Neeratyoy Mallik, Sahithya Ravi, Andreas M\u00fcller, Joaquin Vanschoren, Frank Hutter OpenML-Python: an extensible Python API for OpenML Journal of Machine Learning Research, 22(100):1\u22125, 2021</p> <p>Bibtex entry: <pre><code>@article{JMLR:v22:19-920,\n  author  = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas M\u00fcller and Joaquin Vanschoren and Frank Hutter},\n  title   = {OpenML-Python: an extensible Python API for OpenML},\n  journal = {Journal of Machine Learning Research},\n  year    = {2021},\n  volume  = {22},\n  number  = {100},\n  pages   = {1--5},\n  url     = {http://jmlr.org/papers/v22/19-920.html}\n}\n</code></pre></p>"},{"location":"python/#contributors","title":"Contributors \u2728","text":"<p>Thanks goes to these wonderful people (emoji key):</p> <sub>a-moadel</sub>\ud83d\udcd6 \ud83d\udca1 <sub>Neeratyoy Mallik</sub>\ud83d\udcbb \ud83d\udcd6 \ud83d\udca1 <p>This project follows the all-contributors specification. Contributions of any kind welcome!</p>"},{"location":"python/","title":"OpenML","text":"<p>Collaborative Machine Learning in Python</p> <p>Welcome to the documentation of the OpenML Python API, a connector to the collaborative machine learning platform OpenML.org. The OpenML Python package allows to use datasets and tasks from OpenML together with scikit-learn and share the results online.</p>"},{"location":"python/#example","title":"Example","text":"<pre><code>import openml\nfrom sklearn import impute, tree, pipeline\n\n# Define a scikit-learn classifier or pipeline\nclf = pipeline.Pipeline(\n    steps=[\n        ('imputer', impute.SimpleImputer()),\n        ('estimator', tree.DecisionTreeClassifier())\n    ]\n)\n# Download the OpenML task for the pendigits dataset with 10-fold\n# cross-validation.\ntask = openml.tasks.get_task(32)\n# Run the scikit-learn model on the task.\nrun = openml.runs.run_model_on_task(clf, task)\n# Publish the experiment on OpenML (optional, requires an API key.\n# You can get your own API key by signing up to OpenML.org)\nrun.publish()\nprint(f'View the run online: {run.openml_url}')\n</code></pre> <p>Find more examples in the sidebar on the left.</p>"},{"location":"python/#how-to-get-openml-for-python","title":"How to get OpenML for python","text":"<p>You can install the OpenML package via <code>pip</code> (we recommend using a virtual environment):</p> <pre><code>python -m pip install openml\n</code></pre> <p>For more advanced installation information, please see the \"Introduction\" example.</p>"},{"location":"python/#further-information","title":"Further information","text":"<ul> <li>OpenML documentation</li> <li>OpenML client APIs</li> <li>OpenML developer guide</li> <li>Contact information</li> <li>Citation request</li> <li>OpenML blog</li> <li>OpenML twitter account</li> </ul>"},{"location":"python/#contributing","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated. Please see the \"Contributing\" page for more information.</p>"},{"location":"python/#citing-openml-python","title":"Citing OpenML-Python","text":"<p>If you use OpenML-Python in a scientific publication, we would appreciate a reference to our JMLR-MLOSS paper  \"OpenML-Python: an extensible Python API for OpenML\":</p> BibtexMLA <pre><code>@article{JMLR:v22:19-920,\n    author  = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas M\u00c3\u00bcller and Joaquin Vanschoren and Frank Hutter},\n    title   = {OpenML-Python: an extensible Python API for OpenML},\n    journal = {Journal of Machine Learning Research},\n    year    = {2021},\n    volume  = {22},\n    number  = {100},\n    pages   = {1--5},\n    url     = {http://jmlr.org/papers/v22/19-920.html}\n}\n</code></pre> <p>Feurer, Matthias, et al.  \"OpenML-Python: an extensible Python API for OpenML.\" Journal of Machine Learning Research 22.100 (2021):1\u22125.</p>"},{"location":"python/CONTRIBUTING/","title":"CONTRIBUTING","text":"<p>This document describes the workflow on how to contribute to the openml-python package. If you are interested in connecting a machine learning package with OpenML (i.e. write an openml-python extension) or want to find other ways to contribute, see this page.</p>"},{"location":"python/CONTRIBUTING/#scope-of-the-package","title":"Scope of the package","text":"<p>The scope of the OpenML Python package is to provide a Python interface to the OpenML platform which integrates well with Python's scientific stack, most notably numpy, scipy and pandas. To reduce opportunity costs and demonstrate the usage of the package, it also implements an interface to the most popular machine learning package written in Python, scikit-learn. Thereby it will automatically be compatible with many machine learning libraries written in Python.</p> <p>We aim to keep the package as light-weight as possible and we will try to keep the number of potential installation dependencies as low as possible. Therefore, the connection to other machine learning libraries such as pytorch, keras or tensorflow should not be done directly inside this package, but in a separate package using the OpenML Python connector. More information on OpenML Python connectors can be found here.</p>"},{"location":"python/CONTRIBUTING/#reporting-bugs","title":"Reporting bugs","text":"<p>We use GitHub issues to track all bugs and feature requests; feel free to open an issue if you have found a bug or wish to see a feature implemented.</p> <p>It is recommended to check that your issue complies with the following rules before submitting:</p> <ul> <li> <p>Verify that your issue is not being currently addressed by other    issues    or pull requests.</p> </li> <li> <p>Please ensure all code snippets and error messages are formatted in    appropriate code blocks.    See Creating and highlighting code blocks.</p> </li> <li> <p>Please include your operating system type and version number, as well    as your Python, openml, scikit-learn, numpy, and scipy versions. This information    can be found by running the following code snippet: <pre><code>import platform; print(platform.platform())\nimport sys; print(\"Python\", sys.version)\nimport numpy; print(\"NumPy\", numpy.__version__)\nimport scipy; print(\"SciPy\", scipy.__version__)\nimport sklearn; print(\"Scikit-Learn\", sklearn.__version__)\nimport openml; print(\"OpenML\", openml.__version__)\n</code></pre></p> </li> </ul>"},{"location":"python/CONTRIBUTING/#determine-what-contribution-to-make","title":"Determine what contribution to make","text":"<p>Great! You've decided you want to help out. Now what? All contributions should be linked to issues on the Github issue tracker. In particular for new contributors, the good first issue label should help you find issues which are suitable for beginners.  Resolving these issues allow you to start contributing to the project without much prior knowledge. Your assistance in this area  will be greatly appreciated by the more experienced developers as it helps free up  their time to concentrate on other issues.</p> <p>If you encountered a particular part of the documentation or code that you want to improve, but there is no related open issue yet, open one first. This is important since you can first get feedback or pointers from experienced contributors.</p> <p>To let everyone know you are working on an issue, please leave a comment that states you will work on the issue (or, if you have the permission, assign yourself to the issue). This avoids double work!</p>"},{"location":"python/CONTRIBUTING/#general-git-workflow","title":"General git workflow","text":"<p>The preferred workflow for contributing to openml-python is to fork the main repository on GitHub, clone, check out the branch <code>develop</code>, and develop on a new branch branch. Steps:</p> <ol> <li> <p>Fork the project repository    by clicking on the 'Fork' button near the top right of the page. This creates    a copy of the code under your GitHub user account. For more details on    how to fork a repository see this guide.</p> </li> <li> <p>Clone your fork of the openml-python repo from your GitHub account to your local disk:</p> </li> </ol> <pre><code>$ git clone git@github.com:YourLogin/openml-python.git\n$ cd openml-python\n</code></pre> <ol> <li>Switch to the <code>develop</code> branch:</li> </ol> <pre><code>$ git checkout develop\n</code></pre> <ol> <li>Create a <code>feature</code> branch to hold your development changes:</li> </ol> <pre><code>$ git checkout -b feature/my-feature\n</code></pre> <p>Always use a <code>feature</code> branch. It's good practice to never work on the <code>main</code> or <code>develop</code> branch!     To make the nature of your pull request easily visible, please prepend the name of the branch with the type of changes you want to merge, such as <code>feature</code> if it contains a new feature, <code>fix</code> for a bugfix, <code>doc</code> for documentation and <code>maint</code> for other maintenance on the package.</p> <ol> <li>Develop the feature on your feature branch. Add changed files using <code>git add</code> and then <code>git commit</code> files:</li> </ol> <pre><code>$ git add modified_files\n$ git commit\n</code></pre> <p>to record your changes in Git, then push the changes to your GitHub account with:</p> <pre><code>$ git push -u origin my-feature\n</code></pre> <ol> <li>Follow these instructions to create a pull request from your fork. This will send an email to the committers.</li> </ol> <p>(If any of the above seems like magic to you, please look up the Git documentation on the web, or ask a friend or another contributor for help.)</p>"},{"location":"python/CONTRIBUTING/#pull-request-checklist","title":"Pull Request Checklist","text":"<p>We recommended that your contribution complies with the following rules before you submit a pull request:</p> <ul> <li> <p>Follow the    pep8 style guide.    With the following exceptions or additions:</p> <ul> <li>The max line length is 100 characters instead of 80.</li> <li>When creating a multi-line expression with binary operators, break before the operator.</li> <li>Add type hints to all function signatures. (note: not all functions have type hints yet, this is work in progress.)</li> <li>Use the <code>str.format</code> over <code>printf</code> style formatting.  E.g. use <code>\"{} {}\".format('hello', 'world')</code> not <code>\"%s %s\" % ('hello', 'world')</code>.  (note: old code may still use <code>printf</code>-formatting, this is work in progress.)</li> </ul> </li> <li> <p>If your pull request addresses an issue, please use the pull request title    to describe the issue and mention the issue number in the pull request description. This will make sure a link back to the original issue is    created.</p> </li> <li> <p>An incomplete contribution -- where you expect to do more work before    receiving a full review -- should be submitted as a <code>draft</code>. These may be useful    to: indicate you are working on something to avoid duplicated work,    request broad review of functionality or API, or seek collaborators.    Drafts often benefit from the inclusion of a    task list    in the PR description.</p> </li> <li> <p>Add unit tests and examples for any new functionality being introduced. </p> <ul> <li>If an unit test contains an upload to the test server, please ensure that it is followed by a file collection for deletion, to prevent the test server from bulking up. For example, <code>TestBase._mark_entity_for_removal('data', dataset.dataset_id)</code>, <code>TestBase._mark_entity_for_removal('flow', (flow.flow_id, flow.name))</code>.</li> <li>Please ensure that the example is run on the test server by beginning with the call to <code>openml.config.start_using_configuration_for_example()</code>.</li> <li>Add the <code>@pytest.mark.sklearn</code> marker to your unit tests if they have a dependency on scikit-learn.</li> </ul> </li> <li> <p>All tests pass when running <code>pytest</code>. On    Unix-like systems, check with (from the toplevel source folder):</p> <pre><code>$ pytest\n</code></pre> </li> </ul> <p>For Windows systems, execute the command from an Anaconda Prompt or add <code>pytest</code> to PATH before executing the command.</p> <ul> <li> <p>Documentation and high-coverage tests are necessary for enhancements to be    accepted. Bug-fixes or new features should be provided with    non-regression tests.    These tests verify the correct behavior of the fix or feature. In this    manner, further modifications on the code base are granted to be consistent    with the desired behavior.    For the Bug-fixes case, at the time of the PR, this tests should fail for    the code base in develop and pass for the PR code.</p> </li> <li> <p>Add your changes to the changelog in the file doc/progress.rst.</p> </li> <li> <p>If any source file is being added to the repository, please add the BSD 3-Clause license to it.</p> </li> </ul> <p>Note: We recommend to follow the instructions below to install all requirements locally. However it is also possible to use the openml-python docker image for testing and building documentation. This can be useful for one-off contributions or when you are experiencing installation issues.</p> <p>First install openml with its test dependencies by running   <pre><code>$ pip install -e .[test]\n</code></pre> from the repository folder. Then configure pre-commit through  <pre><code>$ pre-commit install\n</code></pre> This will install dependencies to run unit tests, as well as pre-commit. To run the unit tests, and check their code coverage, run:   <pre><code>$ pytest --cov=. path/to/tests_for_package\n</code></pre> Make sure your code has good unittest coverage (at least 80%).</p> <p>Pre-commit is used for various style checking and code formatting. Before each commit, it will automatically run:  - black a code formatter.    This will automatically format your code.    Make sure to take a second look after any formatting takes place,    if the resulting code is very bloated, consider a (small) refactor.    note: If Black reformats your code, the commit will automatically be aborted.    Make sure to add the formatted files (back) to your commit after checking them.  - mypy a static type checker.    In particular, make sure each function you work on has type hints.  - flake8 style guide enforcement.    Almost all of the black-formatted code should automatically pass this check,    but make sure to make adjustments if it does fail.</p> <p>If you want to run the pre-commit tests without doing a commit, run: <pre><code>$ make check\n</code></pre> or on a system without make, like Windows: <pre><code>$ pre-commit run --all-files\n</code></pre> Make sure to do this at least once before your first commit to check your setup works.</p> <p>Executing a specific unit test can be done by specifying the module, test case, and test. To obtain a hierarchical list of all tests, run</p> <pre><code>$  pytest --collect-only\n\n &lt;Module 'tests/test_datasets/test_dataset.py'&gt;\n   &lt;UnitTestCase 'OpenMLDatasetTest'&gt;\n     &lt;TestCaseFunction 'test_dataset_format_constructor'&gt;\n     &lt;TestCaseFunction 'test_get_data'&gt;\n     &lt;TestCaseFunction 'test_get_data_rowid_and_ignore_and_target'&gt;\n     &lt;TestCaseFunction 'test_get_data_with_ignore_attributes'&gt;\n     &lt;TestCaseFunction 'test_get_data_with_rowid'&gt;\n     &lt;TestCaseFunction 'test_get_data_with_target'&gt;\n   &lt;UnitTestCase 'OpenMLDatasetTestOnTestServer'&gt;\n     &lt;TestCaseFunction 'test_tagging'&gt;\n</code></pre> <p>You may then run a specific module, test case, or unit test respectively: <pre><code>  $ pytest tests/test_datasets/test_dataset.py\n  $ pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest\n  $ pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest::test_get_data\n</code></pre></p> <p>NOTE: In the case the examples build fails during the Continuous Integration test online, please  fix the first failing example. If the first failing example switched the server from live to test  or vice-versa, and the subsequent examples expect the other server, the ensuing examples will fail  to be built as well.</p> <p>Happy testing!</p>"},{"location":"python/CONTRIBUTING/#documentation","title":"Documentation","text":"<p>We are glad to accept any sort of documentation: function docstrings, reStructuredText documents, tutorials, etc. reStructuredText documents live in the source code repository under the doc/ directory.</p> <p>You can edit the documentation using any text editor and then generate the HTML output by typing <code>make html</code> from the doc/ directory. The resulting HTML files will be placed in <code>build/html/</code> and are viewable in a web browser. See the <code>README</code> file in the <code>doc/</code> directory for more information.</p> <p>For building the documentation, you will need to install a few additional dependencies: <pre><code>$ pip install -e .[docs]\n</code></pre> When dependencies are installed, run <pre><code>$ sphinx-build -b html doc YOUR_PREFERRED_OUTPUT_DIRECTORY\n</code></pre></p>"},{"location":"python/ISSUE_TEMPLATE/","title":"ISSUE TEMPLATE","text":""},{"location":"python/ISSUE_TEMPLATE/#description","title":"Description","text":""},{"location":"python/ISSUE_TEMPLATE/#stepscode-to-reproduce","title":"Steps/Code to Reproduce","text":""},{"location":"python/ISSUE_TEMPLATE/#expected-results","title":"Expected Results","text":""},{"location":"python/ISSUE_TEMPLATE/#actual-results","title":"Actual Results","text":""},{"location":"python/ISSUE_TEMPLATE/#versions","title":"Versions","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/","title":"PULL REQUEST TEMPLATE","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#reference-issue","title":"Reference Issue","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#what-does-this-pr-implementfix-explain-your-changes","title":"What does this PR implement/fix? Explain your changes.","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#how-should-this-pr-be-tested","title":"How should this PR be tested?","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#any-other-comments","title":"Any other comments?","text":""},{"location":"python/contributing/","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated in all forms. In particular, a few ways to contribute to openml-python are:</p> <ul> <li>A direct contribution to the package, by means of improving the     code, documentation or examples. To get started, see this     file     with details on how to set up your environment to develop for     openml-python.</li> <li>A contribution to an openml-python extension. An extension package     allows OpenML to interface with a machine learning package (such     as scikit-learn or keras). These extensions are hosted in separate     repositories and may have their own guidelines. For more     information, see also extensions.</li> <li>Bug reports. If something doesn't work for you or is cumbersome,     please open a new issue to let us know about the problem. See     this     section.</li> <li>Cite OpenML if you use it in a     scientific publication.</li> <li>Visit one of our hackathons.</li> <li>Contribute to another OpenML project, such as the main OpenML     project.</li> </ul>"},{"location":"python/extensions/","title":"Extensions","text":"<p>OpenML-Python provides an extension interface to connect other machine learning libraries than scikit-learn to OpenML. Please check the <code>api_extensions</code> and use the scikit-learn extension in <code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text role=\"class\"} as a starting point.</p>"},{"location":"python/extensions/#list-of-extensions","title":"List of extensions","text":"<p>Here is a list of currently maintained OpenML extensions:</p> <ul> <li><code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text     role=\"class\"}</li> <li>openml-keras</li> <li>openml-pytorch</li> <li>openml-tensorflow (for tensorflow     2+)</li> </ul>"},{"location":"python/extensions/#connecting-new-machine-learning-libraries","title":"Connecting new machine learning libraries","text":""},{"location":"python/extensions/#content-of-the-library","title":"Content of the Library","text":"<p>To leverage support from the community and to tap in the potential of OpenML, interfacing with popular machine learning libraries is essential. The OpenML-Python package is capable of downloading meta-data and results (data, flows, runs), regardless of the library that was used to upload it. However, in order to simplify the process of uploading flows and runs from a specific library, an additional interface can be built. The OpenML-Python team does not have the capacity to develop and maintain such interfaces on its own. For this reason, we have built an extension interface to allows others to contribute back. Building a suitable extension for therefore requires an understanding of the current OpenML-Python support.</p> <p>The <code>sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py</code>{.interpreted-text role=\"ref\"} tutorial shows how scikit-learn currently works with OpenML-Python as an extension. The sklearn extension packaged with the openml-python repository can be used as a template/benchmark to build the new extension.</p>"},{"location":"python/extensions/#api","title":"API","text":"<ul> <li>The extension scripts must import the [openml]{.title-ref} package     and be able to interface with any function from the OpenML-Python     <code>api</code>.</li> <li>The extension has to be defined as a Python class and must inherit     from <code>openml.extensions.Extension</code>.</li> <li>This class needs to have all the functions from [class     Extension]{.title-ref} overloaded as required.</li> <li>The redefined functions should have adequate and appropriate     docstrings. The [Sklearn Extension API     :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref}     is a good example to follow.</li> </ul>"},{"location":"python/extensions/#interfacing-with-openml-python","title":"Interfacing with OpenML-Python","text":"<p>Once the new extension class has been defined, the openml-python module to <code>openml.extensions.register_extension</code> must be called to allow OpenML-Python to interface the new extension.</p> <p>The following methods should get implemented. Although the documentation in the [Extension]{.title-ref} interface should always be leading, here we list some additional information and best practices. The [Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is a good example to follow. Note that most methods are relatively simple and can be implemented in several lines of code.</p> <ul> <li>General setup (required)<ul> <li><code>can_handle_flow</code>: Takes as     argument an OpenML flow, and checks whether this can be handled     by the current extension. The OpenML database consists of many     flows, from various workbenches (e.g., scikit-learn, Weka, mlr).     This method is called before a model is being deserialized.     Typically, the flow-dependency field is used to check whether     the specific library is present, and no unknown libraries are     present there.</li> <li><code>can_handle_model</code>: Similar as     <code>can_handle_flow</code>, except that in     this case a Python object is given. As such, in many cases, this     method can be implemented by checking whether this adheres to a     certain base class.</li> </ul> </li> <li>Serialization and De-serialization (required)<ul> <li><code>flow_to_model</code>: deserializes the     OpenML Flow into a model (if the library can indeed handle the     flow). This method has an important interplay with     <code>model_to_flow</code>. Running these     two methods in succession should result in exactly the same     model (or flow). This property can be used for unit testing     (e.g., build a model with hyperparameters, make predictions on a     task, serialize it to a flow, deserialize it back, make it     predict on the same task, and check whether the predictions are     exactly the same.) The example in the scikit-learn interface     might seem daunting, but note that here some complicated design     choices were made, that allow for all sorts of interesting     research questions. It is probably good practice to start easy.</li> <li><code>model_to_flow</code>: The inverse of     <code>flow_to_model</code>. Serializes a     model into an OpenML Flow. The flow should preserve the class,     the library version, and the tunable hyperparameters.</li> <li><code>get_version_information</code>: Return     a tuple with the version information of the important libraries.</li> <li><code>create_setup_string</code>: No longer     used, and will be deprecated soon.</li> </ul> </li> <li>Performing runs (required)<ul> <li><code>is_estimator</code>: Gets as input a     class, and checks whether it has the status of estimator in the     library (typically, whether it has a train method and a predict     method).</li> <li><code>seed_model</code>: Sets a random seed     to the model.</li> <li><code>_run_model_on_fold</code>: One of the     main requirements for a library to generate run objects for the     OpenML server. Obtains a train split (with labels) and a test     split (without labels) and the goal is to train a model on the     train split and return the predictions on the test split. On top     of the actual predictions, also the class probabilities should     be determined. For classifiers that do not return class     probabilities, this can just be the hot-encoded predicted label.     The predictions will be evaluated on the OpenML server. Also,     additional information can be returned, for example,     user-defined measures (such as runtime information, as this can     not be inferred on the server). Additionally, information about     a hyperparameter optimization trace can be provided.</li> <li><code>obtain_parameter_values</code>:     Obtains the hyperparameters of a given model and the current     values. Please note that in the case of a hyperparameter     optimization procedure (e.g., random search), you only should     return the hyperparameters of this procedure (e.g., the     hyperparameter grid, budget, etc) and that the chosen model will     be inferred from the optimization trace.</li> <li><code>check_if_model_fitted</code>: Check     whether the train method of the model has been called (and as     such, whether the predict method can be used).</li> </ul> </li> <li>Hyperparameter optimization (optional)<ul> <li><code>instantiate_model_from_hpo_class</code>{.interpreted-text     role=\"meth\"}: If a given run has recorded the hyperparameter     optimization trace, then this method can be used to     reinstantiate the model with hyperparameters of a given     hyperparameter optimization iteration. Has some similarities     with <code>flow_to_model</code> (as this     method also sets the hyperparameters of a model). Note that     although this method is required, it is not necessary to     implement any logic if hyperparameter optimization is not     implemented. Simply raise a [NotImplementedError]{.title-ref}     then.</li> </ul> </li> </ul>"},{"location":"python/extensions/#hosting-the-library","title":"Hosting the library","text":"<p>Each extension created should be a stand-alone repository, compatible with the OpenML-Python repository. The extension repository should work off-the-shelf with OpenML-Python installed.</p> <p>Create a public Github repo with the following directory structure:</p> <pre><code>| [repo name]\n|    |-- [extension name]\n|    |    |-- __init__.py\n|    |    |-- extension.py\n|    |    |-- config.py (optionally)\n</code></pre>"},{"location":"python/extensions/#recommended","title":"Recommended","text":"<ul> <li>Test cases to keep the extension up to date with the     [openml-python]{.title-ref} upstream changes.</li> <li>Documentation of the extension API, especially if any new     functionality added to OpenML-Python\\'s extension design.</li> <li>Examples to show how the new extension interfaces and works with     OpenML-Python.</li> <li>Create a PR to add the new extension to the OpenML-Python API     documentation.</li> </ul> <p>Happy contributing!</p>"},{"location":"python/progress/","title":"Changelog","text":""},{"location":"python/progress/#next","title":"next","text":"<ul> <li>MAINT #1340: Add Numpy 2.0 support. Update tests to work with     scikit-learn \\&lt;= 1.5.</li> <li>ADD #1342: Add HTTP header to requests to indicate they are from     openml-python.</li> </ul>"},{"location":"python/progress/#0142","title":"0.14.2","text":"<ul> <li>MAINT #1280: Use the server-provided <code>parquet_url</code> instead of     <code>minio_url</code> to determine the location of the parquet file.</li> <li>ADD #716: add documentation for remaining attributes of classes     and functions.</li> <li>ADD #1261: more annotations for type hints.</li> <li>MAINT #1294: update tests to new tag specification.</li> <li>FIX #1314: Update fetching a bucket from MinIO.</li> <li>FIX #1315: Make class label retrieval more lenient.</li> <li>ADD #1316: add feature descriptions ontologies support.</li> <li>MAINT #1310/#1307: switch to ruff and resolve all mypy errors.</li> </ul>"},{"location":"python/progress/#0141","title":"0.14.1","text":"<ul> <li>FIX: Fallback on downloading ARFF when failing to download parquet     from MinIO due to a ServerError.</li> </ul>"},{"location":"python/progress/#0140","title":"0.14.0","text":"<p>IMPORTANT: This release paves the way towards a breaking update of OpenML-Python. From version 0.15, functions that had the option to return a pandas DataFrame will return a pandas DataFrame by default. This version (0.14) emits a warning if you still use the old access functionality. More concretely:</p> <ul> <li>In 0.15 we will drop the ability to return dictionaries in listing     calls and only provide pandas DataFrames. To disable warnings in     0.14 you have to request a pandas DataFrame (using     <code>output_format=\"dataframe\"</code>).</li> <li>In 0.15 we will drop the ability to return datasets as numpy arrays     and only provide pandas DataFrames. To disable warnings in 0.14 you     have to request a pandas DataFrame (using     <code>dataset_format=\"dataframe\"</code>).</li> </ul> <p>Furthermore, from version 0.15, OpenML-Python will no longer download datasets and dataset metadata by default. This version (0.14) emits a warning if you don\\'t explicitly specifiy the desired behavior.</p> <p>Please see the pull requests #1258 and #1260 for further information.</p> <ul> <li>ADD #1081: New flag that allows disabling downloading dataset     features.</li> <li>ADD #1132: New flag that forces a redownload of cached data.</li> <li>FIX #1244: Fixes a rare bug where task listing could fail when the     server returned invalid data.</li> <li>DOC #1229: Fixes a comment string for the main example.</li> <li>DOC #1241: Fixes a comment in an example.</li> <li>MAINT #1124: Improve naming of helper functions that govern the     cache directories.</li> <li>MAINT #1223, #1250: Update tools used in pre-commit to the latest     versions (<code>black==23.30</code>, <code>mypy==1.3.0</code>, <code>flake8==6.0.0</code>).</li> <li>MAINT #1253: Update the citation request to the JMLR paper.</li> <li>MAINT #1246: Add a warning that warns the user that checking for     duplicate runs on the server cannot be done without an API key.</li> </ul>"},{"location":"python/progress/#0131","title":"0.13.1","text":"<ul> <li>ADD #1081 #1132: Add additional options for (not) downloading     datasets <code>openml.datasets.get_dataset</code> and cache management.</li> <li>ADD #1028: Add functions to delete runs, flows, datasets, and tasks     (e.g., <code>openml.datasets.delete_dataset</code>).</li> <li>ADD #1144: Add locally computed results to the <code>OpenMLRun</code> object\\'s     representation if the run was created locally and not downloaded     from the server.</li> <li>ADD #1180: Improve the error message when the checksum of a     downloaded dataset does not match the checksum provided by the API.</li> <li>ADD #1201: Make <code>OpenMLTraceIteration</code> a dataclass.</li> <li>DOC #1069: Add argument documentation for the <code>OpenMLRun</code> class.</li> <li>DOC #1241 #1229 #1231: Minor documentation fixes and resolve     documentation examples not working.</li> <li>FIX #1197 #559 #1131: Fix the order of ground truth and predictions     in the <code>OpenMLRun</code> object and in <code>format_prediction</code>.</li> <li>FIX #1198: Support numpy 1.24 and higher.</li> <li>FIX #1216: Allow unknown task types on the server. This is only     relevant when new task types are added to the test server.</li> <li>FIX #1223: Fix mypy errors for implicit optional typing.</li> <li>MAINT #1155: Add dependabot github action to automatically update     other github actions.</li> <li>MAINT #1199: Obtain pre-commit\\'s flake8 from github.com instead of     gitlab.com.</li> <li>MAINT #1215: Support latest numpy version.</li> <li>MAINT #1218: Test Python3.6 on Ubuntu 20.04 instead of the latest     Ubuntu (which is 22.04).</li> <li>MAINT #1221 #1212 #1206 #1211: Update github actions to the latest     versions.</li> </ul>"},{"location":"python/progress/#0130","title":"0.13.0","text":"<ul> <li>FIX #1030: <code>pre-commit</code> hooks now no longer should issue a     warning.</li> <li>FIX #1058, #1100: Avoid <code>NoneType</code> error when printing task     without <code>class_labels</code> attribute.</li> <li>FIX #1110: Make arguments to <code>create_study</code> and <code>create_suite</code>     that are defined as optional by the OpenML XSD actually optional.</li> <li>FIX #1147: <code>openml.flow.flow_exists</code> no longer requires an API     key.</li> <li>FIX #1184: Automatically resolve proxies when downloading from     minio. Turn this off by setting environment variable     <code>no_proxy=\"*\"</code>.</li> <li>MAINT #1088: Do CI for Windows on Github Actions instead of     Appveyor.</li> <li>MAINT #1104: Fix outdated docstring for <code>list_task</code>.</li> <li>MAINT #1146: Update the pre-commit dependencies.</li> <li>ADD #1103: Add a <code>predictions</code> property to OpenMLRun for easy     accessibility of prediction data.</li> <li>ADD #1188: EXPERIMENTAL. Allow downloading all files from a minio     bucket with <code>download_all_files=True</code> for <code>get_dataset</code>.</li> </ul>"},{"location":"python/progress/#0122","title":"0.12.2","text":"<ul> <li>ADD #1065: Add a <code>retry_policy</code> configuration option that determines     the frequency and number of times to attempt to retry server     requests.</li> <li>ADD #1075: A docker image is now automatically built on a push to     develop. It can be used to build docs or run tests in an isolated     environment.</li> <li>ADD: You can now avoid downloading \\'qualities\\' meta-data when     downloading a task with the <code>download_qualities</code> parameter of     <code>openml.tasks.get_task[s]</code> functions.</li> <li>DOC: Fixes a few broken links in the documentation.</li> <li>DOC #1061: Improve examples to always show a warning when they     switch to the test server.</li> <li>DOC #1067: Improve documentation on the scikit-learn extension     interface.</li> <li>DOC #1068: Create dedicated extensions page.</li> <li>FIX #1075: Correctly convert [y]{.title-ref} to a pandas series when     downloading sparse data.</li> <li>MAINT: Rename [master]{.title-ref} brach to [ main]{.title-ref}     branch.</li> <li>MAINT/DOC: Automatically check for broken external links when     building the documentation.</li> <li>MAINT/DOC: Fail documentation building on warnings. This will make     the documentation building fail if a reference cannot be found (i.e.     an internal link is broken).</li> </ul>"},{"location":"python/progress/#0121","title":"0.12.1","text":"<ul> <li>ADD #895/#1038: Measure runtimes of scikit-learn runs also for     models which are parallelized via the joblib.</li> <li>DOC #1050: Refer to the webpage instead of the XML file in the main     example.</li> <li>DOC #1051: Document existing extensions to OpenML-Python besides the     shipped scikit-learn extension.</li> <li>FIX #1035: Render class attributes and methods again.</li> <li>ADD #1049: Add a command line tool for configuration openml-python.</li> <li>FIX #1042: Fixes a rare concurrency issue with OpenML-Python and     joblib which caused the joblib worker pool to fail.</li> <li>FIX #1053: Fixes a bug which could prevent importing the package in     a docker container.</li> </ul>"},{"location":"python/progress/#0120","title":"0.12.0","text":"<ul> <li>ADD #964: Validate <code>ignore_attribute</code>, <code>default_target_attribute</code>,     <code>row_id_attribute</code> are set to attributes that exist on the dataset     when calling <code>create_dataset</code>.</li> <li>ADD #979: Dataset features and qualities are now also cached in     pickle format.</li> <li>ADD #982: Add helper functions for column transformers.</li> <li>ADD #989: <code>run_model_on_task</code> will now warn the user the the model     passed has already been fitted.</li> <li>ADD #1009 : Give possibility to not download the dataset qualities.     The cached version is used even so download attribute is false.</li> <li>ADD #1016: Add scikit-learn 0.24 support.</li> <li>ADD #1020: Add option to parallelize evaluation of tasks with     joblib.</li> <li>ADD #1022: Allow minimum version of dependencies to be listed for a     flow, use more accurate minimum versions for scikit-learn     dependencies.</li> <li>ADD #1023: Add admin-only calls for adding topics to datasets.</li> <li>ADD #1029: Add support for fetching dataset from a minio server in     parquet format.</li> <li>ADD #1031: Generally improve runtime measurements, add them for some     previously unsupported flows (e.g. BaseSearchCV derived flows).</li> <li>DOC #973 : Change the task used in the welcome page example so it no     longer fails using numerical dataset.</li> <li>MAINT #671: Improved the performance of <code>check_datasets_active</code> by     only querying the given list of datasets in contrast to querying all     datasets. Modified the corresponding unit test.</li> <li>MAINT #891: Changed the way that numerical features are stored.     Numerical features that range from 0 to 255 are now stored as uint8,     which reduces the storage space required as well as storing and     loading times.</li> <li>MAINT #975, #988: Add CI through Github Actions.</li> <li>MAINT #977: Allow <code>short</code> and <code>long</code> scenarios for unit tests.     Reduce the workload for some unit tests.</li> <li>MAINT #985, #1000: Improve unit test stability and output     readability, and adds load balancing.</li> <li>MAINT #1018: Refactor data loading and storage. Data is now     compressed on the first call to [get_data]{.title-ref}.</li> <li>MAINT #1024: Remove flaky decorator for study unit test.</li> <li>FIX #883 #884 #906 #972: Various improvements to the caching system.</li> <li>FIX #980: Speed up <code>check_datasets_active</code>.</li> <li>FIX #984: Add a retry mechanism when the server encounters a     database issue.</li> <li>FIX #1004: Fixed an issue that prevented installation on some     systems (e.g. Ubuntu).</li> <li>FIX #1013: Fixes a bug where <code>OpenMLRun.setup_string</code> was not     uploaded to the server, prepares for <code>run_details</code> being sent from     the server.</li> <li>FIX #1021: Fixes an issue that could occur when running unit tests     and openml-python was not in PATH.</li> <li>FIX #1037: Fixes a bug where a dataset could not be loaded if a     categorical value had listed nan-like as a possible category.</li> </ul>"},{"location":"python/progress/#0110","title":"0.11.0","text":"<ul> <li>ADD #753: Allows uploading custom flows to OpenML via OpenML-Python.</li> <li>ADD #777: Allows running a flow on pandas dataframes (in addition to     numpy arrays).</li> <li>ADD #888: Allow passing a [task_id]{.title-ref} to     [run_model_on_task]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #929: Add <code>edit_dataset</code> and <code>fork_dataset</code> to allow editing and     forking of uploaded datasets.</li> <li>ADD #866, #943: Add support for scikit-learn\\'s     [passthrough]{.title-ref} and [drop]{.title-ref} when uploading     flows to OpenML.</li> <li>ADD #879: Add support for scikit-learn\\'s MLP hyperparameter     [layer_sizes]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #945: PEP 561 compliance for distributing Type information.</li> <li>DOC #660: Remove nonexistent argument from docstring.</li> <li>DOC #901: The API reference now documents the config file and its     options.</li> <li>DOC #912: API reference now shows [create_task]{.title-ref}.</li> <li>DOC #954: Remove TODO text from documentation.</li> <li>DOC #960: document how to upload multiple ignore attributes.</li> <li>FIX #873: Fixes an issue which resulted in incorrect URLs when     printing OpenML objects after switching the server.</li> <li>FIX #885: Logger no longer registered by default. Added utility     functions to easily register logging to console and file.</li> <li>FIX #890: Correct the scaling of data in the SVM example.</li> <li>MAINT #371: <code>list_evaluations</code> default <code>size</code> changed from <code>None</code> to     <code>10_000</code>.</li> <li>MAINT #767: Source distribution installation is now unit-tested.</li> <li>MAINT #781: Add pre-commit and automated code formatting with black.</li> <li>MAINT #804: Rename arguments of list_evaluations to indicate they     expect lists of ids.</li> <li>MAINT #836: OpenML supports only pandas version 1.0.0 or above.</li> <li>MAINT #865: OpenML no longer bundles test files in the source     distribution.</li> <li>MAINT #881: Improve the error message for too-long URIs.</li> <li>MAINT #897: Dropping support for Python 3.5.</li> <li>MAINT #916: Adding support for Python 3.8.</li> <li>MAINT #920: Improve error messages for dataset upload.</li> <li>MAINT #921: Improve hangling of the OpenML server URL in the config     file.</li> <li>MAINT #925: Improve error handling and error message when loading     datasets.</li> <li>MAINT #928: Restructures the contributing documentation.</li> <li>MAINT #936: Adding support for scikit-learn 0.23.X.</li> <li>MAINT #945: Make OpenML-Python PEP562 compliant.</li> <li>MAINT #951: Converts TaskType class to a TaskType enum.</li> </ul>"},{"location":"python/progress/#0102","title":"0.10.2","text":"<ul> <li>ADD #857: Adds task type ID to list_runs</li> <li>DOC #862: Added license BSD 3-Clause to each of the source files.</li> </ul>"},{"location":"python/progress/#0101","title":"0.10.1","text":"<ul> <li>ADD #175: Automatically adds the docstring of scikit-learn objects     to flow and its parameters.</li> <li>ADD #737: New evaluation listing call that includes the     hyperparameter settings.</li> <li>ADD #744: It is now possible to only issue a warning and not raise     an exception if the package versions for a flow are not met when     deserializing it.</li> <li>ADD #783: The URL to download the predictions for a run is now     stored in the run object.</li> <li>ADD #790: Adds the uploader name and id as new filtering options for     <code>list_evaluations</code>.</li> <li>ADD #792: New convenience function <code>openml.flow.get_flow_id</code>.</li> <li>ADD #861: Debug-level log information now being written to a file in     the cache directory (at most 2 MB).</li> <li>DOC #778: Introduces instructions on how to publish an extension to     support other libraries than scikit-learn.</li> <li>DOC #785: The examples section is completely restructured into     simple simple examples, advanced examples and examples showcasing     the use of OpenML-Python to reproduce papers which were done with     OpenML-Python.</li> <li>DOC #788: New example on manually iterating through the split of a     task.</li> <li>DOC #789: Improve the usage of dataframes in the examples.</li> <li>DOC #791: New example for the paper Efficient and Robust Automated     Machine Learning by Feurer et al. (2015).</li> <li>DOC #803: New example for the paper Don't Rule Out Simple Models     Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear     Classifiers in OpenML by Benjamin Strang et al. (2018).</li> <li>DOC #808: New example demonstrating basic use cases of a dataset.</li> <li>DOC #810: New example demonstrating the use of benchmarking studies     and suites.</li> <li>DOC #832: New example for the paper Scalable Hyperparameter     Transfer Learning by Valerio Perrone et al. (2019)</li> <li>DOC #834: New example showing how to plot the loss surface for a     support vector machine.</li> <li>FIX #305: Do not require the external version in the flow XML when     loading an object.</li> <li>FIX #734: Better handling of \\\"old\\\" flows.</li> <li>FIX #736: Attach a StreamHandler to the openml logger instead of the     root logger.</li> <li>FIX #758: Fixes an error which made the client API crash when     loading a sparse data with categorical variables.</li> <li>FIX #779: Do not fail on corrupt pickle</li> <li>FIX #782: Assign the study id to the correct class attribute.</li> <li>FIX #819: Automatically convert column names to type string when     uploading a dataset.</li> <li>FIX #820: Make <code>__repr__</code> work for datasets which do not have an id.</li> <li>MAINT #796: Rename an argument to make the function     <code>list_evaluations</code> more consistent.</li> <li>MAINT #811: Print the full error message given by the server.</li> <li>MAINT #828: Create base class for OpenML entity classes.</li> <li>MAINT #829: Reduce the number of data conversion warnings.</li> <li>MAINT #831: Warn if there\\'s an empty flow description when     publishing a flow.</li> <li>MAINT #837: Also print the flow XML if a flow fails to validate.</li> <li>FIX #838: Fix list_evaluations_setups to work when evaluations are     not a 100 multiple.</li> <li>FIX #847: Fixes an issue where the client API would crash when     trying to download a dataset when there are no qualities available     on the server.</li> <li>MAINT #849: Move logic of most different <code>publish</code> functions into     the base class.</li> <li>MAINt #850: Remove outdated test code.</li> </ul>"},{"location":"python/progress/#0100","title":"0.10.0","text":"<ul> <li>ADD #737: Add list_evaluations_setups to return hyperparameters     along with list of evaluations.</li> <li>FIX #261: Test server is cleared of all files uploaded during unit     testing.</li> <li>FIX #447: All files created by unit tests no longer persist in     local.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>FIX #447: All files created by unit tests are deleted after the     completion of all unit tests.</li> <li>FIX #589: Fixing a bug that did not successfully upload the columns     to ignore when creating and publishing a dataset.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>DOC #639: More descriptive documention for function to convert array     format.</li> <li>DOC #719: Add documentation on uploading tasks.</li> <li>ADD #687: Adds a function to retrieve the list of evaluation     measures available.</li> <li>ADD #695: A function to retrieve all the data quality measures     available.</li> <li>ADD #412: Add a function to trim flow names for scikit-learn flows.</li> <li>ADD #715: [list_evaluations]{.title-ref} now has an option to sort     evaluations by score (value).</li> <li>ADD #722: Automatic reinstantiation of flow in     [run_model_on_task]{.title-ref}. Clearer errors if that\\'s not     possible.</li> <li>ADD #412: The scikit-learn extension populates the short name field     for flows.</li> <li>MAINT #726: Update examples to remove deprecation warnings from     scikit-learn</li> <li>MAINT #752: Update OpenML-Python to be compatible with sklearn 0.21</li> <li>ADD #790: Add user ID and name to list_evaluations</li> </ul>"},{"location":"python/progress/#090","title":"0.9.0","text":"<ul> <li>ADD #560: OpenML-Python can now handle regression tasks as well.</li> <li>ADD #620, #628, #632, #649, #682: Full support for studies and     distinguishes suites from studies.</li> <li>ADD #607: Tasks can now be created and uploaded.</li> <li>ADD #647, #673: Introduced the extension interface. This provides an     easy way to create a hook for machine learning packages to perform     e.g. automated runs.</li> <li>ADD #548, #646, #676: Support for Pandas DataFrame and     SparseDataFrame</li> <li>ADD #662: Results of listing functions can now be returned as     pandas.DataFrame.</li> <li>ADD #59: Datasets can now also be retrieved by name.</li> <li>ADD #672: Add timing measurements for runs, when possible.</li> <li>ADD #661: Upload time and error messages now displayed with     [list_runs]{.title-ref}.</li> <li>ADD #644: Datasets can now be downloaded \\'lazily\\', retrieving only     metadata at first, and the full dataset only when necessary.</li> <li>ADD #659: Lazy loading of task splits.</li> <li>ADD #516: [run_flow_on_task]{.title-ref} flow uploading is now     optional.</li> <li>ADD #680: Adds     [openml.config.start_using_configuration_for_example]{.title-ref}     (and resp. stop) to easily connect to the test server.</li> <li>ADD #75, #653: Adds a pretty print for objects of the top-level     classes.</li> <li>FIX #642: [check_datasets_active]{.title-ref} now correctly also     returns active status of deactivated datasets.</li> <li>FIX #304, #636: Allow serialization of numpy datatypes and list of     lists of more types (e.g. bools, ints) for flows.</li> <li>FIX #651: Fixed a bug that would prevent openml-python from finding     the user\\'s config file.</li> <li>FIX #693: OpenML-Python uses liac-arff instead of scipy.io for     loading task splits now.</li> <li>DOC #678: Better color scheme for code examples in documentation.</li> <li>DOC #681: Small improvements and removing list of missing functions.</li> <li>DOC #684: Add notice to examples that connect to the test server.</li> <li>DOC #688: Add new example on retrieving evaluations.</li> <li>DOC #691: Update contributing guidelines to use Github draft feature     instead of tags in title.</li> <li>DOC #692: All functions are documented now.</li> <li>MAINT #184: Dropping Python2 support.</li> <li>MAINT #596: Fewer dependencies for regular pip install.</li> <li>MAINT #652: Numpy and Scipy are no longer required before     installation.</li> <li>MAINT #655: Lazy loading is now preferred in unit tests.</li> <li>MAINT #667: Different tag functions now share code.</li> <li>MAINT #666: More descriptive error message for     [TypeError]{.title-ref} in [list_runs]{.title-ref}.</li> <li>MAINT #668: Fix some type hints.</li> <li>MAINT #677: [dataset.get_data]{.title-ref} now has consistent     behavior in its return type.</li> <li>MAINT #686: Adds ignore directives for several [mypy]{.title-ref}     folders.</li> <li>MAINT #629, #630: Code now adheres to single PEP8 standard.</li> </ul>"},{"location":"python/progress/#080","title":"0.8.0","text":"<ul> <li>ADD #440: Improved dataset upload.</li> <li>ADD #545, #583: Allow uploading a dataset from a pandas DataFrame.</li> <li>ADD #528: New functions to update the status of a dataset.</li> <li>ADD #523: Support for scikit-learn 0.20\\'s new ColumnTransformer.</li> <li>ADD #459: Enhanced support to store runs on disk prior to uploading     them to OpenML.</li> <li>ADD #564: New helpers to access the structure of a flow (and find     its subflows).</li> <li>ADD #618: The software will from now on retry to connect to the     server if a connection failed. The number of retries can be     configured.</li> <li>FIX #538: Support loading clustering tasks.</li> <li>FIX #464: Fixes a bug related to listing functions (returns correct     listing size).</li> <li>FIX #580: Listing function now works properly when there are less     results than requested.</li> <li>FIX #571: Fixes an issue where tasks could not be downloaded in     parallel.</li> <li>FIX #536: Flows can now be printed when the flow name is None.</li> <li>FIX #504: Better support for hierarchical hyperparameters when     uploading scikit-learn\\'s grid and random search.</li> <li>FIX #569: Less strict checking of flow dependencies when loading     flows.</li> <li>FIX #431: Pickle of task splits are no longer cached.</li> <li>DOC #540: More examples for dataset uploading.</li> <li>DOC #554: Remove the doubled progress entry from the docs.</li> <li>MAINT #613: Utilize the latest updates in OpenML evaluation     listings.</li> <li>MAINT #482: Cleaner interface for handling search traces.</li> <li>MAINT #557: Continuous integration works for scikit-learn 0.18-0.20.</li> <li>MAINT #542: Continuous integration now runs python3.7 as well.</li> <li>MAINT #535: Continuous integration now enforces PEP8 compliance for     new code.</li> <li>MAINT #527: Replace deprecated nose by pytest.</li> <li>MAINT #510: Documentation is now built by travis-ci instead of     circle-ci.</li> <li>MAINT: Completely re-designed documentation built on sphinx gallery.</li> <li>MAINT #462: Appveyor CI support.</li> <li>MAINT #477: Improve error handling for issue     #479: the OpenML     connector fails earlier and with a better error message when failing     to create a flow from the OpenML description.</li> <li>MAINT #561: Improve documentation on running specific unit tests.</li> </ul>"},{"location":"python/progress/#04-07","title":"0.4.-0.7","text":"<p>There is no changelog for these versions.</p>"},{"location":"python/progress/#030","title":"0.3.0","text":"<ul> <li>Add this changelog</li> <li>2<sup>nd</sup> example notebook PyOpenML.ipynb</li> <li>Pagination support for list datasets and list tasks</li> </ul>"},{"location":"python/progress/#prior","title":"Prior","text":"<p>There is no changelog for prior versions.</p>"},{"location":"python/usage/","title":"User Guide","text":"<p>This document will guide you through the most important use cases, functions and classes in the OpenML Python API. Throughout this document, we will use pandas to format and filter tables.</p>"},{"location":"python/usage/#installation","title":"Installation","text":"<p>The OpenML Python package is a connector to OpenML. It allows you to use and share datasets and tasks, run machine learning algorithms on them and then share the results online.</p> <p>The \"intruduction tutorial and setup\" tutorial gives a short introduction on how to install and set up the OpenML Python connector, followed up by a simple example.</p>"},{"location":"python/usage/#configuration","title":"Configuration","text":"<p>The configuration file resides in a directory <code>.config/openml</code> in the home directory of the user and is called config (More specifically, it resides in the configuration directory specified by the XDGB Base Directory Specification). It consists of <code>key = value</code> pairs which are separated by newlines. The following keys are defined:</p> <ul> <li>apikey: required to access the server. The introduction tutorial describes how to obtain an API key.</li> <li>server: the server to connect to (default: <code>http://www.openml.org</code>).           For connection to the test server, set this to <code>test.openml.org</code>.</li> <li>cachedir: the root folder where the cache file directories should be created.     If not given, will default to <code>~/.openml/cache</code></li> <li>avoid_duplicate_runs: if set to <code>True</code> (default), when <code>run_flow_on_task</code> or similar methods             are called a lookup is performed to see if there already             exists such a run on the server. If so, download those             results instead.</li> <li> <p>retry_policy: Defines how to react when the server is unavailable or             experiencing high load. It determines both how often to             attempt to reconnect and how quickly to do so. Please don't             use <code>human</code> in an automated script that you run more than             one instance of, it might increase the time to complete your             jobs and that of others. One of:             -   human (default): For people running openml in interactive                 fashion. Try only a few times, but in quick succession.             -   robot: For people using openml in an automated fashion. Keep                 trying to reconnect for a longer time, quickly increasing                 the time between retries.</p> </li> <li> <p>connection_n_retries: number of times to retry a request if they fail.  Default depends on retry_policy (5 for <code>human</code>, 50 for <code>robot</code>)</p> </li> <li>verbosity: the level of output:<ul> <li>0: normal output</li> <li>1: info output</li> <li>2: debug output</li> </ul> </li> </ul> <p>This file is easily configurable by the <code>openml</code> command line interface. To see where the file is stored, and what its values are, use openml configure none. </p>"},{"location":"python/usage/#docker","title":"Docker","text":"<p>It is also possible to try out the latest development version of <code>openml-python</code> with docker:</p> <pre><code>docker run -it openml/openml-python\n</code></pre> <p>See the openml-python docker documentation for more information.</p>"},{"location":"python/usage/#key-concepts","title":"Key concepts","text":"<p>OpenML contains several key concepts which it needs to make machine learning research shareable. A machine learning experiment consists of one or several runs, which describe the performance of an algorithm (called a flow in OpenML), its hyperparameter settings (called a setup) on a task. A Task is the combination of a dataset, a split and an evaluation metric. In this user guide we will go through listing and exploring existing tasks to actually running machine learning algorithms on them. In a further user guide we will examine how to search through datasets in order to curate a list of tasks.</p> <p>A further explanation is given in the OpenML user guide.</p>"},{"location":"python/usage/#working-with-tasks","title":"Working with tasks","text":"<p>You can think of a task as an experimentation protocol, describing how to apply a machine learning model to a dataset in a way that is comparable with the results of others (more on how to do that further down). Tasks are containers, defining which dataset to use, what kind of task we\\'re solving (regression, classification, clustering, etc...) and which column to predict. Furthermore, it also describes how to split the dataset into a train and test set, whether to use several disjoint train and test splits (cross-validation) and whether this should be repeated several times. Also, the task defines a target metric for which a flow should be optimized.</p> <p>If you want to know more about tasks, try the \"Task tutorial\"</p>"},{"location":"python/usage/#running-machine-learning-algorithms-and-uploading-results","title":"Running machine learning algorithms and uploading results","text":"<p>In order to upload and share results of running a machine learning algorithm on a task, we need to create an openml.runs.OpenMLRun. A run object can be created by running a openml.flows.OpenMLFlow or a scikit-learn compatible model on a task. We will focus on the simpler example of running a scikit-learn model.</p> <p>Flows are descriptions of something runnable which does the machine learning. A flow contains all information to set up the necessary machine learning library and its dependencies as well as all possible parameters.</p> <p>A run is the outcome of running a flow on a task. It contains all parameter settings for the flow, a setup string (most likely a command line call) and all predictions of that run. When a run is uploaded to the server, the server automatically calculates several metrics which can be used to compare the performance of different flows to each other.</p> <p>So far, the OpenML Python connector works only with estimator objects following the scikit-learn estimator API. Those can be directly run on a task, and a flow will automatically be created or downloaded from the server if it already exists.</p> <p>See \"Simple Flows and Runs\" for a tutorial covers how to train different machine learning models, how to run machine learning models on OpenML data and how to share the results.</p>"},{"location":"python/usage/#datasets","title":"Datasets","text":"<p>OpenML provides a large collection of datasets and the benchmark OpenML100 which consists of a curated list of datasets.</p> <p>You can find the dataset that best fits your requirements by making use of the available metadata. The tutorial \"extended datasets\" which follows explains how to get a list of datasets, how to filter the list to find the dataset that suits your requirements and how to download a dataset.</p> <p>OpenML is about sharing machine learning results and the datasets they were obtained on. Learn how to share your datasets in the following tutorial \"Upload\" tutorial.</p>"},{"location":"python/usage/#extending-openml-python","title":"Extending OpenML-Python","text":"<p>OpenML-Python provides an extension interface to connect machine learning libraries directly to the API and ships a <code>scikit-learn</code> extension. Read more about them in the \"Extensions\" section.</p>"},{"location":"python/docker/readme/","title":"OpenML Python Container","text":"<p>This docker container has the latest version of openml-python downloaded and pre-installed. It can also be used by developers to run unit tests or build the docs in  a fresh and/or isolated unix environment.  This document contains information about:</p> <ol> <li>Usage: how to use the image and its main modes.</li> <li>Using local or remote code: useful when testing your own latest changes.</li> <li>Versions: identify which image to use.</li> <li>Development: information about the Docker image for developers.</li> </ol> <p>note: each docker image is shipped with a readme, which you can read with: <code>docker run --entrypoint=/bin/cat openml/openml-python:TAG readme.md</code></p>"},{"location":"python/docker/readme/#usage","title":"Usage","text":"<p>There are three main ways to use the image: running a pre-installed Python environment, running tests, and building documentation.</p>"},{"location":"python/docker/readme/#running-python-with-pre-installed-openml-python-default","title":"Running <code>Python</code> with pre-installed <code>OpenML-Python</code> (default):","text":"<p>To run <code>Python</code> with a pre-installed <code>OpenML-Python</code> environment run:</p> <pre><code>docker run -it openml/openml-python\n</code></pre> <p>this accepts the normal <code>Python</code> arguments, e.g.:</p> <pre><code>docker run openml/openml-python -c \"import openml; print(openml.__version__)\"\n</code></pre> <p>if you want to run a local script, it needs to be mounted first. Mount it into the <code>openml</code> folder:</p> <pre><code>docker run -v PATH/TO/FILE:/openml/MY_SCRIPT.py openml/openml-python MY_SCRIPT.py\n</code></pre>"},{"location":"python/docker/readme/#running-unit-tests","title":"Running unit tests","text":"<p>You can run the unit tests by passing <code>test</code> as the first argument. It also requires a local or remote repository to be specified, which is explained  [below]((#using-local-or-remote-code). For this example, we specify to test the <code>develop</code> branch:</p> <pre><code>docker run openml/openml-python test develop\n</code></pre>"},{"location":"python/docker/readme/#building-documentation","title":"Building documentation","text":"<p>You can build the documentation by passing <code>doc</code> as the first argument,  you should mount  an output directory in which the docs will be stored. You also need to provide a remote or local repository as explained in [the section below]((#using-local-or-remote-code). In this example, we build documentation for the <code>develop</code> branch. On Windows:</p> <pre><code>    docker run --mount type=bind,source=\"E:\\\\files/output\",destination=\"/output\" openml/openml-python doc develop\n</code></pre> <p>on Linux: <pre><code>    docker run --mount type=bind,source=\"./output\",destination=\"/output\" openml/openml-python doc develop\n</code></pre></p> <p>see [the section below]((#using-local-or-remote-code) for running against local changes or a remote branch.</p> <p>Note: you can forgo mounting an output directory to test if the docs build successfully, but the result will only be available within the docker container under <code>/openml/docs/build</code>.</p>"},{"location":"python/docker/readme/#using-local-or-remote-code","title":"Using local or remote code","text":"<p>You can build docs or run tests against your local repository or a Github repository. In the examples below, change the <code>source</code> to match the location of your local repository.</p>"},{"location":"python/docker/readme/#using-a-local-repository","title":"Using a local repository","text":"<p>To use a local directory, mount it in the <code>/code</code> directory,  on Windows:</p> <pre><code>    docker run --mount type=bind,source=\"E:\\\\repositories/openml-python\",destination=\"/code\" openml/openml-python test\n</code></pre> <p>on Linux: <pre><code>    docker run --mount type=bind,source=\"/Users/pietergijsbers/repositories/openml-python\",destination=\"/code\" openml/openml-python test\n</code></pre></p> <p>when building docs, you also need to mount an output directory as shown above, so add both:</p> <pre><code>docker run --mount type=bind,source=\"./output\",destination=\"/output\" --mount type=bind,source=\"/Users/pietergijsbers/repositories/openml-python\",destination=\"/code\" openml/openml-python doc\n</code></pre>"},{"location":"python/docker/readme/#using-a-github-repository","title":"Using a Github repository","text":"<p>Building from a remote repository requires you to specify a branch. The branch may be specified by name directly if it exists on the original repository (https://github.com/openml/openml-python/):</p> <pre><code>docker run --mount type=bind,source=PATH_TO_OUTPUT,destination=/output openml/openml-python [test,doc] BRANCH\n</code></pre> <p>Where <code>BRANCH</code> is the name of the branch for which to generate the documentation. It is also possible to build the documentation from the branch on a fork, in this case the <code>BRANCH</code> should be specified as <code>GITHUB_NAME#BRANCH</code> (e.g.  <code>PGijsbers#my_feature_branch</code>) and the name of the forked repository should be <code>openml-python</code>.</p>"},{"location":"python/docker/readme/#for-developers","title":"For developers","text":"<p>This section contains some notes about the structure of the image,  intended for those who want to work on it.</p>"},{"location":"python/docker/readme/#added-directories","title":"Added Directories","text":"<p>The <code>openml/openml-python</code> image is built on a vanilla <code>python:3</code> image. Additionally, it contains the following files are directories:</p> <ul> <li><code>/openml</code>: contains the openml-python repository in the state with which the image     was built by default. If working with a <code>BRANCH</code>, this repository will be set to     the <code>HEAD</code> of <code>BRANCH</code>.</li> <li><code>/openml/venv/</code>: contains the used virtual environment for <code>doc</code> and <code>test</code>. It has    <code>openml-python</code> dependencies pre-installed.  When invoked with <code>doc</code> or <code>test</code>, the     dependencies will be updated based on the <code>setup.py</code> of the <code>BRANCH</code> or mounted <code>/code</code>.</li> <li><code>/scripts/startup.sh</code>: the entrypoint of the image. Takes care of the automated features (e.g. <code>doc</code> and <code>test</code>).</li> </ul>"},{"location":"python/docker/readme/#building-the-image","title":"Building the image","text":"<p>To build the image yourself, execute <code>docker build -f Dockerfile .</code> from the <code>docker</code> directory of the <code>openml-python</code> repository. It will use the <code>startup.sh</code> as is, so any  local changes will be present in the image.</p>"},{"location":"python/docs/","title":"OpenML","text":"<p>Collaborative Machine Learning in Python</p> <p>Welcome to the documentation of the OpenML Python API, a connector to the collaborative machine learning platform OpenML.org. The OpenML Python package allows to use datasets and tasks from OpenML together with scikit-learn and share the results online.</p>"},{"location":"python/docs/#example","title":"Example","text":"<pre><code>import openml\nfrom sklearn import impute, tree, pipeline\n\n# Define a scikit-learn classifier or pipeline\nclf = pipeline.Pipeline(\n    steps=[\n        ('imputer', impute.SimpleImputer()),\n        ('estimator', tree.DecisionTreeClassifier())\n    ]\n)\n# Download the OpenML task for the pendigits dataset with 10-fold\n# cross-validation.\ntask = openml.tasks.get_task(32)\n# Run the scikit-learn model on the task.\nrun = openml.runs.run_model_on_task(clf, task)\n# Publish the experiment on OpenML (optional, requires an API key.\n# You can get your own API key by signing up to OpenML.org)\nrun.publish()\nprint(f'View the run online: {run.openml_url}')\n</code></pre> <p>Find more examples in the sidebar on the left.</p>"},{"location":"python/docs/#how-to-get-openml-for-python","title":"How to get OpenML for python","text":"<p>You can install the OpenML package via <code>pip</code> (we recommend using a virtual environment):</p> <pre><code>python -m pip install openml\n</code></pre> <p>For more advanced installation information, please see the \"Introduction\" example.</p>"},{"location":"python/docs/#further-information","title":"Further information","text":"<ul> <li>OpenML documentation</li> <li>OpenML client APIs</li> <li>OpenML developer guide</li> <li>Contact information</li> <li>Citation request</li> <li>OpenML blog</li> <li>OpenML twitter account</li> </ul>"},{"location":"python/docs/#contributing","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated. Please see the \"Contributing\" page for more information.</p>"},{"location":"python/docs/#citing-openml-python","title":"Citing OpenML-Python","text":"<p>If you use OpenML-Python in a scientific publication, we would appreciate a reference to our JMLR-MLOSS paper  \"OpenML-Python: an extensible Python API for OpenML\":</p> BibtexMLA <pre><code>@article{JMLR:v22:19-920,\n    author  = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas M\u00c3\u00bcller and Joaquin Vanschoren and Frank Hutter},\n    title   = {OpenML-Python: an extensible Python API for OpenML},\n    journal = {Journal of Machine Learning Research},\n    year    = {2021},\n    volume  = {22},\n    number  = {100},\n    pages   = {1--5},\n    url     = {http://jmlr.org/papers/v22/19-920.html}\n}\n</code></pre> <p>Feurer, Matthias, et al.  \"OpenML-Python: an extensible Python API for OpenML.\" Journal of Machine Learning Research 22.100 (2021):1\u22125.</p>"},{"location":"python/docs/contributing/","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated in all forms. In particular, a few ways to contribute to openml-python are:</p> <ul> <li>A direct contribution to the package, by means of improving the     code, documentation or examples. To get started, see this     file     with details on how to set up your environment to develop for     openml-python.</li> <li>A contribution to an openml-python extension. An extension package     allows OpenML to interface with a machine learning package (such     as scikit-learn or keras). These extensions are hosted in separate     repositories and may have their own guidelines. For more     information, see also extensions.</li> <li>Bug reports. If something doesn't work for you or is cumbersome,     please open a new issue to let us know about the problem. See     this     section.</li> <li>Cite OpenML if you use it in a     scientific publication.</li> <li>Visit one of our hackathons.</li> <li>Contribute to another OpenML project, such as the main OpenML     project.</li> </ul>"},{"location":"python/docs/extensions/","title":"Extensions","text":"<p>OpenML-Python provides an extension interface to connect other machine learning libraries than scikit-learn to OpenML. Please check the <code>api_extensions</code> and use the scikit-learn extension in <code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text role=\"class\"} as a starting point.</p>"},{"location":"python/docs/extensions/#list-of-extensions","title":"List of extensions","text":"<p>Here is a list of currently maintained OpenML extensions:</p> <ul> <li><code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text     role=\"class\"}</li> <li>openml-keras</li> <li>openml-pytorch</li> <li>openml-tensorflow (for tensorflow     2+)</li> </ul>"},{"location":"python/docs/extensions/#connecting-new-machine-learning-libraries","title":"Connecting new machine learning libraries","text":""},{"location":"python/docs/extensions/#content-of-the-library","title":"Content of the Library","text":"<p>To leverage support from the community and to tap in the potential of OpenML, interfacing with popular machine learning libraries is essential. The OpenML-Python package is capable of downloading meta-data and results (data, flows, runs), regardless of the library that was used to upload it. However, in order to simplify the process of uploading flows and runs from a specific library, an additional interface can be built. The OpenML-Python team does not have the capacity to develop and maintain such interfaces on its own. For this reason, we have built an extension interface to allows others to contribute back. Building a suitable extension for therefore requires an understanding of the current OpenML-Python support.</p> <p>The <code>sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py</code>{.interpreted-text role=\"ref\"} tutorial shows how scikit-learn currently works with OpenML-Python as an extension. The sklearn extension packaged with the openml-python repository can be used as a template/benchmark to build the new extension.</p>"},{"location":"python/docs/extensions/#api","title":"API","text":"<ul> <li>The extension scripts must import the [openml]{.title-ref} package     and be able to interface with any function from the OpenML-Python     <code>api</code>.</li> <li>The extension has to be defined as a Python class and must inherit     from <code>openml.extensions.Extension</code>.</li> <li>This class needs to have all the functions from [class     Extension]{.title-ref} overloaded as required.</li> <li>The redefined functions should have adequate and appropriate     docstrings. The [Sklearn Extension API     :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref}     is a good example to follow.</li> </ul>"},{"location":"python/docs/extensions/#interfacing-with-openml-python","title":"Interfacing with OpenML-Python","text":"<p>Once the new extension class has been defined, the openml-python module to <code>openml.extensions.register_extension</code> must be called to allow OpenML-Python to interface the new extension.</p> <p>The following methods should get implemented. Although the documentation in the [Extension]{.title-ref} interface should always be leading, here we list some additional information and best practices. The [Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is a good example to follow. Note that most methods are relatively simple and can be implemented in several lines of code.</p> <ul> <li>General setup (required)<ul> <li><code>can_handle_flow</code>: Takes as     argument an OpenML flow, and checks whether this can be handled     by the current extension. The OpenML database consists of many     flows, from various workbenches (e.g., scikit-learn, Weka, mlr).     This method is called before a model is being deserialized.     Typically, the flow-dependency field is used to check whether     the specific library is present, and no unknown libraries are     present there.</li> <li><code>can_handle_model</code>: Similar as     <code>can_handle_flow</code>, except that in     this case a Python object is given. As such, in many cases, this     method can be implemented by checking whether this adheres to a     certain base class.</li> </ul> </li> <li>Serialization and De-serialization (required)<ul> <li><code>flow_to_model</code>: deserializes the     OpenML Flow into a model (if the library can indeed handle the     flow). This method has an important interplay with     <code>model_to_flow</code>. Running these     two methods in succession should result in exactly the same     model (or flow). This property can be used for unit testing     (e.g., build a model with hyperparameters, make predictions on a     task, serialize it to a flow, deserialize it back, make it     predict on the same task, and check whether the predictions are     exactly the same.) The example in the scikit-learn interface     might seem daunting, but note that here some complicated design     choices were made, that allow for all sorts of interesting     research questions. It is probably good practice to start easy.</li> <li><code>model_to_flow</code>: The inverse of     <code>flow_to_model</code>. Serializes a     model into an OpenML Flow. The flow should preserve the class,     the library version, and the tunable hyperparameters.</li> <li><code>get_version_information</code>: Return     a tuple with the version information of the important libraries.</li> <li><code>create_setup_string</code>: No longer     used, and will be deprecated soon.</li> </ul> </li> <li>Performing runs (required)<ul> <li><code>is_estimator</code>: Gets as input a     class, and checks whether it has the status of estimator in the     library (typically, whether it has a train method and a predict     method).</li> <li><code>seed_model</code>: Sets a random seed     to the model.</li> <li><code>_run_model_on_fold</code>: One of the     main requirements for a library to generate run objects for the     OpenML server. Obtains a train split (with labels) and a test     split (without labels) and the goal is to train a model on the     train split and return the predictions on the test split. On top     of the actual predictions, also the class probabilities should     be determined. For classifiers that do not return class     probabilities, this can just be the hot-encoded predicted label.     The predictions will be evaluated on the OpenML server. Also,     additional information can be returned, for example,     user-defined measures (such as runtime information, as this can     not be inferred on the server). Additionally, information about     a hyperparameter optimization trace can be provided.</li> <li><code>obtain_parameter_values</code>:     Obtains the hyperparameters of a given model and the current     values. Please note that in the case of a hyperparameter     optimization procedure (e.g., random search), you only should     return the hyperparameters of this procedure (e.g., the     hyperparameter grid, budget, etc) and that the chosen model will     be inferred from the optimization trace.</li> <li><code>check_if_model_fitted</code>: Check     whether the train method of the model has been called (and as     such, whether the predict method can be used).</li> </ul> </li> <li>Hyperparameter optimization (optional)<ul> <li><code>instantiate_model_from_hpo_class</code>{.interpreted-text     role=\"meth\"}: If a given run has recorded the hyperparameter     optimization trace, then this method can be used to     reinstantiate the model with hyperparameters of a given     hyperparameter optimization iteration. Has some similarities     with <code>flow_to_model</code> (as this     method also sets the hyperparameters of a model). Note that     although this method is required, it is not necessary to     implement any logic if hyperparameter optimization is not     implemented. Simply raise a [NotImplementedError]{.title-ref}     then.</li> </ul> </li> </ul>"},{"location":"python/docs/extensions/#hosting-the-library","title":"Hosting the library","text":"<p>Each extension created should be a stand-alone repository, compatible with the OpenML-Python repository. The extension repository should work off-the-shelf with OpenML-Python installed.</p> <p>Create a public Github repo with the following directory structure:</p> <pre><code>| [repo name]\n|    |-- [extension name]\n|    |    |-- __init__.py\n|    |    |-- extension.py\n|    |    |-- config.py (optionally)\n</code></pre>"},{"location":"python/docs/extensions/#recommended","title":"Recommended","text":"<ul> <li>Test cases to keep the extension up to date with the     [openml-python]{.title-ref} upstream changes.</li> <li>Documentation of the extension API, especially if any new     functionality added to OpenML-Python\\'s extension design.</li> <li>Examples to show how the new extension interfaces and works with     OpenML-Python.</li> <li>Create a PR to add the new extension to the OpenML-Python API     documentation.</li> </ul> <p>Happy contributing!</p>"},{"location":"python/docs/progress/","title":"Changelog","text":""},{"location":"python/docs/progress/#next","title":"next","text":"<ul> <li>MAINT #1340: Add Numpy 2.0 support. Update tests to work with     scikit-learn \\&lt;= 1.5.</li> <li>ADD #1342: Add HTTP header to requests to indicate they are from     openml-python.</li> </ul>"},{"location":"python/docs/progress/#0142","title":"0.14.2","text":"<ul> <li>MAINT #1280: Use the server-provided <code>parquet_url</code> instead of     <code>minio_url</code> to determine the location of the parquet file.</li> <li>ADD #716: add documentation for remaining attributes of classes     and functions.</li> <li>ADD #1261: more annotations for type hints.</li> <li>MAINT #1294: update tests to new tag specification.</li> <li>FIX #1314: Update fetching a bucket from MinIO.</li> <li>FIX #1315: Make class label retrieval more lenient.</li> <li>ADD #1316: add feature descriptions ontologies support.</li> <li>MAINT #1310/#1307: switch to ruff and resolve all mypy errors.</li> </ul>"},{"location":"python/docs/progress/#0141","title":"0.14.1","text":"<ul> <li>FIX: Fallback on downloading ARFF when failing to download parquet     from MinIO due to a ServerError.</li> </ul>"},{"location":"python/docs/progress/#0140","title":"0.14.0","text":"<p>IMPORTANT: This release paves the way towards a breaking update of OpenML-Python. From version 0.15, functions that had the option to return a pandas DataFrame will return a pandas DataFrame by default. This version (0.14) emits a warning if you still use the old access functionality. More concretely:</p> <ul> <li>In 0.15 we will drop the ability to return dictionaries in listing     calls and only provide pandas DataFrames. To disable warnings in     0.14 you have to request a pandas DataFrame (using     <code>output_format=\"dataframe\"</code>).</li> <li>In 0.15 we will drop the ability to return datasets as numpy arrays     and only provide pandas DataFrames. To disable warnings in 0.14 you     have to request a pandas DataFrame (using     <code>dataset_format=\"dataframe\"</code>).</li> </ul> <p>Furthermore, from version 0.15, OpenML-Python will no longer download datasets and dataset metadata by default. This version (0.14) emits a warning if you don\\'t explicitly specifiy the desired behavior.</p> <p>Please see the pull requests #1258 and #1260 for further information.</p> <ul> <li>ADD #1081: New flag that allows disabling downloading dataset     features.</li> <li>ADD #1132: New flag that forces a redownload of cached data.</li> <li>FIX #1244: Fixes a rare bug where task listing could fail when the     server returned invalid data.</li> <li>DOC #1229: Fixes a comment string for the main example.</li> <li>DOC #1241: Fixes a comment in an example.</li> <li>MAINT #1124: Improve naming of helper functions that govern the     cache directories.</li> <li>MAINT #1223, #1250: Update tools used in pre-commit to the latest     versions (<code>black==23.30</code>, <code>mypy==1.3.0</code>, <code>flake8==6.0.0</code>).</li> <li>MAINT #1253: Update the citation request to the JMLR paper.</li> <li>MAINT #1246: Add a warning that warns the user that checking for     duplicate runs on the server cannot be done without an API key.</li> </ul>"},{"location":"python/docs/progress/#0131","title":"0.13.1","text":"<ul> <li>ADD #1081 #1132: Add additional options for (not) downloading     datasets <code>openml.datasets.get_dataset</code> and cache management.</li> <li>ADD #1028: Add functions to delete runs, flows, datasets, and tasks     (e.g., <code>openml.datasets.delete_dataset</code>).</li> <li>ADD #1144: Add locally computed results to the <code>OpenMLRun</code> object\\'s     representation if the run was created locally and not downloaded     from the server.</li> <li>ADD #1180: Improve the error message when the checksum of a     downloaded dataset does not match the checksum provided by the API.</li> <li>ADD #1201: Make <code>OpenMLTraceIteration</code> a dataclass.</li> <li>DOC #1069: Add argument documentation for the <code>OpenMLRun</code> class.</li> <li>DOC #1241 #1229 #1231: Minor documentation fixes and resolve     documentation examples not working.</li> <li>FIX #1197 #559 #1131: Fix the order of ground truth and predictions     in the <code>OpenMLRun</code> object and in <code>format_prediction</code>.</li> <li>FIX #1198: Support numpy 1.24 and higher.</li> <li>FIX #1216: Allow unknown task types on the server. This is only     relevant when new task types are added to the test server.</li> <li>FIX #1223: Fix mypy errors for implicit optional typing.</li> <li>MAINT #1155: Add dependabot github action to automatically update     other github actions.</li> <li>MAINT #1199: Obtain pre-commit\\'s flake8 from github.com instead of     gitlab.com.</li> <li>MAINT #1215: Support latest numpy version.</li> <li>MAINT #1218: Test Python3.6 on Ubuntu 20.04 instead of the latest     Ubuntu (which is 22.04).</li> <li>MAINT #1221 #1212 #1206 #1211: Update github actions to the latest     versions.</li> </ul>"},{"location":"python/docs/progress/#0130","title":"0.13.0","text":"<ul> <li>FIX #1030: <code>pre-commit</code> hooks now no longer should issue a     warning.</li> <li>FIX #1058, #1100: Avoid <code>NoneType</code> error when printing task     without <code>class_labels</code> attribute.</li> <li>FIX #1110: Make arguments to <code>create_study</code> and <code>create_suite</code>     that are defined as optional by the OpenML XSD actually optional.</li> <li>FIX #1147: <code>openml.flow.flow_exists</code> no longer requires an API     key.</li> <li>FIX #1184: Automatically resolve proxies when downloading from     minio. Turn this off by setting environment variable     <code>no_proxy=\"*\"</code>.</li> <li>MAINT #1088: Do CI for Windows on Github Actions instead of     Appveyor.</li> <li>MAINT #1104: Fix outdated docstring for <code>list_task</code>.</li> <li>MAINT #1146: Update the pre-commit dependencies.</li> <li>ADD #1103: Add a <code>predictions</code> property to OpenMLRun for easy     accessibility of prediction data.</li> <li>ADD #1188: EXPERIMENTAL. Allow downloading all files from a minio     bucket with <code>download_all_files=True</code> for <code>get_dataset</code>.</li> </ul>"},{"location":"python/docs/progress/#0122","title":"0.12.2","text":"<ul> <li>ADD #1065: Add a <code>retry_policy</code> configuration option that determines     the frequency and number of times to attempt to retry server     requests.</li> <li>ADD #1075: A docker image is now automatically built on a push to     develop. It can be used to build docs or run tests in an isolated     environment.</li> <li>ADD: You can now avoid downloading \\'qualities\\' meta-data when     downloading a task with the <code>download_qualities</code> parameter of     <code>openml.tasks.get_task[s]</code> functions.</li> <li>DOC: Fixes a few broken links in the documentation.</li> <li>DOC #1061: Improve examples to always show a warning when they     switch to the test server.</li> <li>DOC #1067: Improve documentation on the scikit-learn extension     interface.</li> <li>DOC #1068: Create dedicated extensions page.</li> <li>FIX #1075: Correctly convert [y]{.title-ref} to a pandas series when     downloading sparse data.</li> <li>MAINT: Rename [master]{.title-ref} brach to [ main]{.title-ref}     branch.</li> <li>MAINT/DOC: Automatically check for broken external links when     building the documentation.</li> <li>MAINT/DOC: Fail documentation building on warnings. This will make     the documentation building fail if a reference cannot be found (i.e.     an internal link is broken).</li> </ul>"},{"location":"python/docs/progress/#0121","title":"0.12.1","text":"<ul> <li>ADD #895/#1038: Measure runtimes of scikit-learn runs also for     models which are parallelized via the joblib.</li> <li>DOC #1050: Refer to the webpage instead of the XML file in the main     example.</li> <li>DOC #1051: Document existing extensions to OpenML-Python besides the     shipped scikit-learn extension.</li> <li>FIX #1035: Render class attributes and methods again.</li> <li>ADD #1049: Add a command line tool for configuration openml-python.</li> <li>FIX #1042: Fixes a rare concurrency issue with OpenML-Python and     joblib which caused the joblib worker pool to fail.</li> <li>FIX #1053: Fixes a bug which could prevent importing the package in     a docker container.</li> </ul>"},{"location":"python/docs/progress/#0120","title":"0.12.0","text":"<ul> <li>ADD #964: Validate <code>ignore_attribute</code>, <code>default_target_attribute</code>,     <code>row_id_attribute</code> are set to attributes that exist on the dataset     when calling <code>create_dataset</code>.</li> <li>ADD #979: Dataset features and qualities are now also cached in     pickle format.</li> <li>ADD #982: Add helper functions for column transformers.</li> <li>ADD #989: <code>run_model_on_task</code> will now warn the user the the model     passed has already been fitted.</li> <li>ADD #1009 : Give possibility to not download the dataset qualities.     The cached version is used even so download attribute is false.</li> <li>ADD #1016: Add scikit-learn 0.24 support.</li> <li>ADD #1020: Add option to parallelize evaluation of tasks with     joblib.</li> <li>ADD #1022: Allow minimum version of dependencies to be listed for a     flow, use more accurate minimum versions for scikit-learn     dependencies.</li> <li>ADD #1023: Add admin-only calls for adding topics to datasets.</li> <li>ADD #1029: Add support for fetching dataset from a minio server in     parquet format.</li> <li>ADD #1031: Generally improve runtime measurements, add them for some     previously unsupported flows (e.g. BaseSearchCV derived flows).</li> <li>DOC #973 : Change the task used in the welcome page example so it no     longer fails using numerical dataset.</li> <li>MAINT #671: Improved the performance of <code>check_datasets_active</code> by     only querying the given list of datasets in contrast to querying all     datasets. Modified the corresponding unit test.</li> <li>MAINT #891: Changed the way that numerical features are stored.     Numerical features that range from 0 to 255 are now stored as uint8,     which reduces the storage space required as well as storing and     loading times.</li> <li>MAINT #975, #988: Add CI through Github Actions.</li> <li>MAINT #977: Allow <code>short</code> and <code>long</code> scenarios for unit tests.     Reduce the workload for some unit tests.</li> <li>MAINT #985, #1000: Improve unit test stability and output     readability, and adds load balancing.</li> <li>MAINT #1018: Refactor data loading and storage. Data is now     compressed on the first call to [get_data]{.title-ref}.</li> <li>MAINT #1024: Remove flaky decorator for study unit test.</li> <li>FIX #883 #884 #906 #972: Various improvements to the caching system.</li> <li>FIX #980: Speed up <code>check_datasets_active</code>.</li> <li>FIX #984: Add a retry mechanism when the server encounters a     database issue.</li> <li>FIX #1004: Fixed an issue that prevented installation on some     systems (e.g. Ubuntu).</li> <li>FIX #1013: Fixes a bug where <code>OpenMLRun.setup_string</code> was not     uploaded to the server, prepares for <code>run_details</code> being sent from     the server.</li> <li>FIX #1021: Fixes an issue that could occur when running unit tests     and openml-python was not in PATH.</li> <li>FIX #1037: Fixes a bug where a dataset could not be loaded if a     categorical value had listed nan-like as a possible category.</li> </ul>"},{"location":"python/docs/progress/#0110","title":"0.11.0","text":"<ul> <li>ADD #753: Allows uploading custom flows to OpenML via OpenML-Python.</li> <li>ADD #777: Allows running a flow on pandas dataframes (in addition to     numpy arrays).</li> <li>ADD #888: Allow passing a [task_id]{.title-ref} to     [run_model_on_task]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #929: Add <code>edit_dataset</code> and <code>fork_dataset</code> to allow editing and     forking of uploaded datasets.</li> <li>ADD #866, #943: Add support for scikit-learn\\'s     [passthrough]{.title-ref} and [drop]{.title-ref} when uploading     flows to OpenML.</li> <li>ADD #879: Add support for scikit-learn\\'s MLP hyperparameter     [layer_sizes]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #945: PEP 561 compliance for distributing Type information.</li> <li>DOC #660: Remove nonexistent argument from docstring.</li> <li>DOC #901: The API reference now documents the config file and its     options.</li> <li>DOC #912: API reference now shows [create_task]{.title-ref}.</li> <li>DOC #954: Remove TODO text from documentation.</li> <li>DOC #960: document how to upload multiple ignore attributes.</li> <li>FIX #873: Fixes an issue which resulted in incorrect URLs when     printing OpenML objects after switching the server.</li> <li>FIX #885: Logger no longer registered by default. Added utility     functions to easily register logging to console and file.</li> <li>FIX #890: Correct the scaling of data in the SVM example.</li> <li>MAINT #371: <code>list_evaluations</code> default <code>size</code> changed from <code>None</code> to     <code>10_000</code>.</li> <li>MAINT #767: Source distribution installation is now unit-tested.</li> <li>MAINT #781: Add pre-commit and automated code formatting with black.</li> <li>MAINT #804: Rename arguments of list_evaluations to indicate they     expect lists of ids.</li> <li>MAINT #836: OpenML supports only pandas version 1.0.0 or above.</li> <li>MAINT #865: OpenML no longer bundles test files in the source     distribution.</li> <li>MAINT #881: Improve the error message for too-long URIs.</li> <li>MAINT #897: Dropping support for Python 3.5.</li> <li>MAINT #916: Adding support for Python 3.8.</li> <li>MAINT #920: Improve error messages for dataset upload.</li> <li>MAINT #921: Improve hangling of the OpenML server URL in the config     file.</li> <li>MAINT #925: Improve error handling and error message when loading     datasets.</li> <li>MAINT #928: Restructures the contributing documentation.</li> <li>MAINT #936: Adding support for scikit-learn 0.23.X.</li> <li>MAINT #945: Make OpenML-Python PEP562 compliant.</li> <li>MAINT #951: Converts TaskType class to a TaskType enum.</li> </ul>"},{"location":"python/docs/progress/#0102","title":"0.10.2","text":"<ul> <li>ADD #857: Adds task type ID to list_runs</li> <li>DOC #862: Added license BSD 3-Clause to each of the source files.</li> </ul>"},{"location":"python/docs/progress/#0101","title":"0.10.1","text":"<ul> <li>ADD #175: Automatically adds the docstring of scikit-learn objects     to flow and its parameters.</li> <li>ADD #737: New evaluation listing call that includes the     hyperparameter settings.</li> <li>ADD #744: It is now possible to only issue a warning and not raise     an exception if the package versions for a flow are not met when     deserializing it.</li> <li>ADD #783: The URL to download the predictions for a run is now     stored in the run object.</li> <li>ADD #790: Adds the uploader name and id as new filtering options for     <code>list_evaluations</code>.</li> <li>ADD #792: New convenience function <code>openml.flow.get_flow_id</code>.</li> <li>ADD #861: Debug-level log information now being written to a file in     the cache directory (at most 2 MB).</li> <li>DOC #778: Introduces instructions on how to publish an extension to     support other libraries than scikit-learn.</li> <li>DOC #785: The examples section is completely restructured into     simple simple examples, advanced examples and examples showcasing     the use of OpenML-Python to reproduce papers which were done with     OpenML-Python.</li> <li>DOC #788: New example on manually iterating through the split of a     task.</li> <li>DOC #789: Improve the usage of dataframes in the examples.</li> <li>DOC #791: New example for the paper Efficient and Robust Automated     Machine Learning by Feurer et al. (2015).</li> <li>DOC #803: New example for the paper Don't Rule Out Simple Models     Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear     Classifiers in OpenML by Benjamin Strang et al. (2018).</li> <li>DOC #808: New example demonstrating basic use cases of a dataset.</li> <li>DOC #810: New example demonstrating the use of benchmarking studies     and suites.</li> <li>DOC #832: New example for the paper Scalable Hyperparameter     Transfer Learning by Valerio Perrone et al. (2019)</li> <li>DOC #834: New example showing how to plot the loss surface for a     support vector machine.</li> <li>FIX #305: Do not require the external version in the flow XML when     loading an object.</li> <li>FIX #734: Better handling of \\\"old\\\" flows.</li> <li>FIX #736: Attach a StreamHandler to the openml logger instead of the     root logger.</li> <li>FIX #758: Fixes an error which made the client API crash when     loading a sparse data with categorical variables.</li> <li>FIX #779: Do not fail on corrupt pickle</li> <li>FIX #782: Assign the study id to the correct class attribute.</li> <li>FIX #819: Automatically convert column names to type string when     uploading a dataset.</li> <li>FIX #820: Make <code>__repr__</code> work for datasets which do not have an id.</li> <li>MAINT #796: Rename an argument to make the function     <code>list_evaluations</code> more consistent.</li> <li>MAINT #811: Print the full error message given by the server.</li> <li>MAINT #828: Create base class for OpenML entity classes.</li> <li>MAINT #829: Reduce the number of data conversion warnings.</li> <li>MAINT #831: Warn if there\\'s an empty flow description when     publishing a flow.</li> <li>MAINT #837: Also print the flow XML if a flow fails to validate.</li> <li>FIX #838: Fix list_evaluations_setups to work when evaluations are     not a 100 multiple.</li> <li>FIX #847: Fixes an issue where the client API would crash when     trying to download a dataset when there are no qualities available     on the server.</li> <li>MAINT #849: Move logic of most different <code>publish</code> functions into     the base class.</li> <li>MAINt #850: Remove outdated test code.</li> </ul>"},{"location":"python/docs/progress/#0100","title":"0.10.0","text":"<ul> <li>ADD #737: Add list_evaluations_setups to return hyperparameters     along with list of evaluations.</li> <li>FIX #261: Test server is cleared of all files uploaded during unit     testing.</li> <li>FIX #447: All files created by unit tests no longer persist in     local.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>FIX #447: All files created by unit tests are deleted after the     completion of all unit tests.</li> <li>FIX #589: Fixing a bug that did not successfully upload the columns     to ignore when creating and publishing a dataset.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>DOC #639: More descriptive documention for function to convert array     format.</li> <li>DOC #719: Add documentation on uploading tasks.</li> <li>ADD #687: Adds a function to retrieve the list of evaluation     measures available.</li> <li>ADD #695: A function to retrieve all the data quality measures     available.</li> <li>ADD #412: Add a function to trim flow names for scikit-learn flows.</li> <li>ADD #715: [list_evaluations]{.title-ref} now has an option to sort     evaluations by score (value).</li> <li>ADD #722: Automatic reinstantiation of flow in     [run_model_on_task]{.title-ref}. Clearer errors if that\\'s not     possible.</li> <li>ADD #412: The scikit-learn extension populates the short name field     for flows.</li> <li>MAINT #726: Update examples to remove deprecation warnings from     scikit-learn</li> <li>MAINT #752: Update OpenML-Python to be compatible with sklearn 0.21</li> <li>ADD #790: Add user ID and name to list_evaluations</li> </ul>"},{"location":"python/docs/progress/#090","title":"0.9.0","text":"<ul> <li>ADD #560: OpenML-Python can now handle regression tasks as well.</li> <li>ADD #620, #628, #632, #649, #682: Full support for studies and     distinguishes suites from studies.</li> <li>ADD #607: Tasks can now be created and uploaded.</li> <li>ADD #647, #673: Introduced the extension interface. This provides an     easy way to create a hook for machine learning packages to perform     e.g. automated runs.</li> <li>ADD #548, #646, #676: Support for Pandas DataFrame and     SparseDataFrame</li> <li>ADD #662: Results of listing functions can now be returned as     pandas.DataFrame.</li> <li>ADD #59: Datasets can now also be retrieved by name.</li> <li>ADD #672: Add timing measurements for runs, when possible.</li> <li>ADD #661: Upload time and error messages now displayed with     [list_runs]{.title-ref}.</li> <li>ADD #644: Datasets can now be downloaded \\'lazily\\', retrieving only     metadata at first, and the full dataset only when necessary.</li> <li>ADD #659: Lazy loading of task splits.</li> <li>ADD #516: [run_flow_on_task]{.title-ref} flow uploading is now     optional.</li> <li>ADD #680: Adds     [openml.config.start_using_configuration_for_example]{.title-ref}     (and resp. stop) to easily connect to the test server.</li> <li>ADD #75, #653: Adds a pretty print for objects of the top-level     classes.</li> <li>FIX #642: [check_datasets_active]{.title-ref} now correctly also     returns active status of deactivated datasets.</li> <li>FIX #304, #636: Allow serialization of numpy datatypes and list of     lists of more types (e.g. bools, ints) for flows.</li> <li>FIX #651: Fixed a bug that would prevent openml-python from finding     the user\\'s config file.</li> <li>FIX #693: OpenML-Python uses liac-arff instead of scipy.io for     loading task splits now.</li> <li>DOC #678: Better color scheme for code examples in documentation.</li> <li>DOC #681: Small improvements and removing list of missing functions.</li> <li>DOC #684: Add notice to examples that connect to the test server.</li> <li>DOC #688: Add new example on retrieving evaluations.</li> <li>DOC #691: Update contributing guidelines to use Github draft feature     instead of tags in title.</li> <li>DOC #692: All functions are documented now.</li> <li>MAINT #184: Dropping Python2 support.</li> <li>MAINT #596: Fewer dependencies for regular pip install.</li> <li>MAINT #652: Numpy and Scipy are no longer required before     installation.</li> <li>MAINT #655: Lazy loading is now preferred in unit tests.</li> <li>MAINT #667: Different tag functions now share code.</li> <li>MAINT #666: More descriptive error message for     [TypeError]{.title-ref} in [list_runs]{.title-ref}.</li> <li>MAINT #668: Fix some type hints.</li> <li>MAINT #677: [dataset.get_data]{.title-ref} now has consistent     behavior in its return type.</li> <li>MAINT #686: Adds ignore directives for several [mypy]{.title-ref}     folders.</li> <li>MAINT #629, #630: Code now adheres to single PEP8 standard.</li> </ul>"},{"location":"python/docs/progress/#080","title":"0.8.0","text":"<ul> <li>ADD #440: Improved dataset upload.</li> <li>ADD #545, #583: Allow uploading a dataset from a pandas DataFrame.</li> <li>ADD #528: New functions to update the status of a dataset.</li> <li>ADD #523: Support for scikit-learn 0.20\\'s new ColumnTransformer.</li> <li>ADD #459: Enhanced support to store runs on disk prior to uploading     them to OpenML.</li> <li>ADD #564: New helpers to access the structure of a flow (and find     its subflows).</li> <li>ADD #618: The software will from now on retry to connect to the     server if a connection failed. The number of retries can be     configured.</li> <li>FIX #538: Support loading clustering tasks.</li> <li>FIX #464: Fixes a bug related to listing functions (returns correct     listing size).</li> <li>FIX #580: Listing function now works properly when there are less     results than requested.</li> <li>FIX #571: Fixes an issue where tasks could not be downloaded in     parallel.</li> <li>FIX #536: Flows can now be printed when the flow name is None.</li> <li>FIX #504: Better support for hierarchical hyperparameters when     uploading scikit-learn\\'s grid and random search.</li> <li>FIX #569: Less strict checking of flow dependencies when loading     flows.</li> <li>FIX #431: Pickle of task splits are no longer cached.</li> <li>DOC #540: More examples for dataset uploading.</li> <li>DOC #554: Remove the doubled progress entry from the docs.</li> <li>MAINT #613: Utilize the latest updates in OpenML evaluation     listings.</li> <li>MAINT #482: Cleaner interface for handling search traces.</li> <li>MAINT #557: Continuous integration works for scikit-learn 0.18-0.20.</li> <li>MAINT #542: Continuous integration now runs python3.7 as well.</li> <li>MAINT #535: Continuous integration now enforces PEP8 compliance for     new code.</li> <li>MAINT #527: Replace deprecated nose by pytest.</li> <li>MAINT #510: Documentation is now built by travis-ci instead of     circle-ci.</li> <li>MAINT: Completely re-designed documentation built on sphinx gallery.</li> <li>MAINT #462: Appveyor CI support.</li> <li>MAINT #477: Improve error handling for issue     #479: the OpenML     connector fails earlier and with a better error message when failing     to create a flow from the OpenML description.</li> <li>MAINT #561: Improve documentation on running specific unit tests.</li> </ul>"},{"location":"python/docs/progress/#04-07","title":"0.4.-0.7","text":"<p>There is no changelog for these versions.</p>"},{"location":"python/docs/progress/#030","title":"0.3.0","text":"<ul> <li>Add this changelog</li> <li>2<sup>nd</sup> example notebook PyOpenML.ipynb</li> <li>Pagination support for list datasets and list tasks</li> </ul>"},{"location":"python/docs/progress/#prior","title":"Prior","text":"<p>There is no changelog for prior versions.</p>"},{"location":"python/docs/usage/","title":"User Guide","text":"<p>This document will guide you through the most important use cases, functions and classes in the OpenML Python API. Throughout this document, we will use pandas to format and filter tables.</p>"},{"location":"python/docs/usage/#installation","title":"Installation","text":"<p>The OpenML Python package is a connector to OpenML. It allows you to use and share datasets and tasks, run machine learning algorithms on them and then share the results online.</p> <p>The \"intruduction tutorial and setup\" tutorial gives a short introduction on how to install and set up the OpenML Python connector, followed up by a simple example.</p>"},{"location":"python/docs/usage/#configuration","title":"Configuration","text":"<p>The configuration file resides in a directory <code>.config/openml</code> in the home directory of the user and is called config (More specifically, it resides in the configuration directory specified by the XDGB Base Directory Specification). It consists of <code>key = value</code> pairs which are separated by newlines. The following keys are defined:</p> <ul> <li>apikey: required to access the server. The introduction tutorial describes how to obtain an API key.</li> <li>server: the server to connect to (default: <code>http://www.openml.org</code>).           For connection to the test server, set this to <code>test.openml.org</code>.</li> <li>cachedir: the root folder where the cache file directories should be created.     If not given, will default to <code>~/.openml/cache</code></li> <li>avoid_duplicate_runs: if set to <code>True</code> (default), when <code>run_flow_on_task</code> or similar methods             are called a lookup is performed to see if there already             exists such a run on the server. If so, download those             results instead.</li> <li> <p>retry_policy: Defines how to react when the server is unavailable or             experiencing high load. It determines both how often to             attempt to reconnect and how quickly to do so. Please don't             use <code>human</code> in an automated script that you run more than             one instance of, it might increase the time to complete your             jobs and that of others. One of:             -   human (default): For people running openml in interactive                 fashion. Try only a few times, but in quick succession.             -   robot: For people using openml in an automated fashion. Keep                 trying to reconnect for a longer time, quickly increasing                 the time between retries.</p> </li> <li> <p>connection_n_retries: number of times to retry a request if they fail.  Default depends on retry_policy (5 for <code>human</code>, 50 for <code>robot</code>)</p> </li> <li>verbosity: the level of output:<ul> <li>0: normal output</li> <li>1: info output</li> <li>2: debug output</li> </ul> </li> </ul> <p>This file is easily configurable by the <code>openml</code> command line interface. To see where the file is stored, and what its values are, use openml configure none. </p>"},{"location":"python/docs/usage/#docker","title":"Docker","text":"<p>It is also possible to try out the latest development version of <code>openml-python</code> with docker:</p> <pre><code>docker run -it openml/openml-python\n</code></pre> <p>See the openml-python docker documentation for more information.</p>"},{"location":"python/docs/usage/#key-concepts","title":"Key concepts","text":"<p>OpenML contains several key concepts which it needs to make machine learning research shareable. A machine learning experiment consists of one or several runs, which describe the performance of an algorithm (called a flow in OpenML), its hyperparameter settings (called a setup) on a task. A Task is the combination of a dataset, a split and an evaluation metric. In this user guide we will go through listing and exploring existing tasks to actually running machine learning algorithms on them. In a further user guide we will examine how to search through datasets in order to curate a list of tasks.</p> <p>A further explanation is given in the OpenML user guide.</p>"},{"location":"python/docs/usage/#working-with-tasks","title":"Working with tasks","text":"<p>You can think of a task as an experimentation protocol, describing how to apply a machine learning model to a dataset in a way that is comparable with the results of others (more on how to do that further down). Tasks are containers, defining which dataset to use, what kind of task we\\'re solving (regression, classification, clustering, etc...) and which column to predict. Furthermore, it also describes how to split the dataset into a train and test set, whether to use several disjoint train and test splits (cross-validation) and whether this should be repeated several times. Also, the task defines a target metric for which a flow should be optimized.</p> <p>If you want to know more about tasks, try the \"Task tutorial\"</p>"},{"location":"python/docs/usage/#running-machine-learning-algorithms-and-uploading-results","title":"Running machine learning algorithms and uploading results","text":"<p>In order to upload and share results of running a machine learning algorithm on a task, we need to create an openml.runs.OpenMLRun. A run object can be created by running a openml.flows.OpenMLFlow or a scikit-learn compatible model on a task. We will focus on the simpler example of running a scikit-learn model.</p> <p>Flows are descriptions of something runnable which does the machine learning. A flow contains all information to set up the necessary machine learning library and its dependencies as well as all possible parameters.</p> <p>A run is the outcome of running a flow on a task. It contains all parameter settings for the flow, a setup string (most likely a command line call) and all predictions of that run. When a run is uploaded to the server, the server automatically calculates several metrics which can be used to compare the performance of different flows to each other.</p> <p>So far, the OpenML Python connector works only with estimator objects following the scikit-learn estimator API. Those can be directly run on a task, and a flow will automatically be created or downloaded from the server if it already exists.</p> <p>See \"Simple Flows and Runs\" for a tutorial covers how to train different machine learning models, how to run machine learning models on OpenML data and how to share the results.</p>"},{"location":"python/docs/usage/#datasets","title":"Datasets","text":"<p>OpenML provides a large collection of datasets and the benchmark OpenML100 which consists of a curated list of datasets.</p> <p>You can find the dataset that best fits your requirements by making use of the available metadata. The tutorial \"extended datasets\" which follows explains how to get a list of datasets, how to filter the list to find the dataset that suits your requirements and how to download a dataset.</p> <p>OpenML is about sharing machine learning results and the datasets they were obtained on. Learn how to share your datasets in the following tutorial \"Upload\" tutorial.</p>"},{"location":"python/docs/usage/#extending-openml-python","title":"Extending OpenML-Python","text":"<p>OpenML-Python provides an extension interface to connect machine learning libraries directly to the API and ships a <code>scikit-learn</code> extension. Read more about them in the \"Extensions\" section.</p>"},{"location":"pytorch/","title":"Pytorch extension for OpenML python","text":"<p>Pytorch extension for openml-python API.</p>"},{"location":"pytorch/#installation-instructions","title":"Installation Instructions:","text":"<p><code>pip install openml-pytorch</code></p> <p>PyPi link https://pypi.org/project/openml-pytorch/</p>"},{"location":"pytorch/#usage","title":"Usage","text":"<p>To use this extension, you need to have a task from OpenML. You can either browse the OpenML website to find a task (and get it's ID), or follow the example to create a task from a custom dataset.</p> <p>Then, follow one of the examples in the Examples folder to see how to use this extension for your type of data.</p> <p>Import openML libraries <pre><code>import torch.nn\nimport torch.optim\n\nimport openml_pytorch.config\nimport openml\nimport logging\n\nfrom openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda\nimport torchvision\nfrom openml_pytorch.trainer import convert_to_rgb\n</code></pre> Create a pytorch model and get a task from openML <pre><code>model = torchvision.models.efficientnet_b0(num_classes=200)\n# Download the OpenML task for tiniest imagenet\ntask = openml.tasks.get_task(362128)\n</code></pre> Download the task from openML and define Data and Trainer configuration <pre><code>transform = Compose(\n    [\n        ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n        Lambda(\n            convert_to_rgb\n        ),  # Convert PIL Image to RGB if it's not already.\n        Resize(\n            (64, 64)\n        ),  # Resize the image.\n        ToTensor(),  # Convert the PIL Image back to a tensor.\n    ]\n)\ndata_module = OpenMLDataModule(\n    type_of_data=\"image\",\n    file_dir=\"datasets\",\n    filename_col=\"image_path\",\n    target_mode=\"categorical\",\n    target_column=\"label\",\n    batch_size = 64,\n    transform=transform\n)\ntrainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n)\nopenml_pytorch.config.trainer = trainer\n</code></pre> Run the model on the task <pre><code>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\nrun.publish()\nprint('URL for run: %s/run/%d' % (openml.config.server, run.run_id))\n</code></pre> Note: The input layer of the network should be compatible with OpenML data output shape. Please check examples for more information.</p> <p>Additionally, if you want to publish the run with onnx file, then you must call <code>openml_pytorch.add_onnx_to_run()</code> immediately before <code>run.publish()</code>. </p> <pre><code>run = openml_pytorch.add_onnx_to_run(run)\n</code></pre>"},{"location":"pytorch/Limitations%20of%20the%20API/","title":"Limitations","text":"<ul> <li>Image datasets are supported as a workaround by using a CSV file with image paths. This is not ideal and might eventually be replaced by something else. At the moment, the focus is on tabular data.</li> <li>Many features (like custom metrics, models etc) are still dependant on the OpenML Python API, which is in the middle of a major rewrite. Until that is complete, this package will not be able to provide all the features it aims to.</li> </ul>"},{"location":"pytorch/Philosophy%20behind%20the%20API%20Design/","title":"Philosophy behind the API design","text":"<p>This API is designed to make it easier to use PyTorch with OpenML and has been heavily inspired by the current state of the art Deep Learning frameworks like FastAI and PyTorch Lightning. </p> <p>To make the library as modular as possible, callbacks are used throughout the training loop. This allows for easy customization of the training loop without having to modify the core code.</p>"},{"location":"pytorch/Philosophy%20behind%20the%20API%20Design/#separation-of-concerns","title":"Separation of Concerns","text":"<p>Here, we focus on the data, model and training as separate blocks that can be strung together in a pipeline. This makes it easier to experiment with different models, data and training strategies.</p> <p>That being the case, the OpenMLDataModule and OpenMLTrainerModule are designed to handle the data and training respectively. This might seem a bit verbose at first, but it makes it easier to understand what is happening at each step of the process and allows for easier customization.</p>"},{"location":"pytorch/API%20reference/Callbacks/","title":"Callbacks","text":"<p>Callbacks module contains classes and functions for handling callback functions during an event-driven process. This makes it easier to customize the behavior of the training loop and add additional functionality to the training process without modifying the core code.</p> <p>To use a callback, create a class that inherits from the Callback class and implement the necessary methods. Callbacks can be used to perform actions at different stages of the training process, such as at the beginning or end of an epoch, batch, or fitting process. Then pass the callback object to the Trainer.</p>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.AvgStats","title":"<code>AvgStats</code>","text":"<p>AvgStats class is used to track and accumulate average statistics (like loss and other metrics) during training and validation phases.</p> <p>Attributes:     metrics (list): A list of metric functions to be tracked.     in_train (bool): A flag to indicate if the statistics are for the training phase.</p> <p>Methods:     init(metrics, in_train):         Initializes the AvgStats with metrics and in_train flag.</p> <pre><code>reset():\n    Resets the accumulated statistics.\n\nall_stats:\n    Property that returns all accumulated statistics including loss and metrics.\n\navg_stats:\n    Property that returns the average of the accumulated statistics.\n\naccumulate(run):\n    Accumulates the statistics using the data from the given run.\n\n__repr__():\n    Returns a string representation of the average statistics.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class AvgStats:\n    \"\"\"\n    AvgStats class is used to track and accumulate average statistics (like loss and other metrics) during training and validation phases.\n\n    Attributes:\n        metrics (list): A list of metric functions to be tracked.\n        in_train (bool): A flag to indicate if the statistics are for the training phase.\n\n    Methods:\n        __init__(metrics, in_train):\n            Initializes the AvgStats with metrics and in_train flag.\n\n        reset():\n            Resets the accumulated statistics.\n\n        all_stats:\n            Property that returns all accumulated statistics including loss and metrics.\n\n        avg_stats:\n            Property that returns the average of the accumulated statistics.\n\n        accumulate(run):\n            Accumulates the statistics using the data from the given run.\n\n        __repr__():\n            Returns a string representation of the average statistics.\n    \"\"\"\n    def __init__(self, metrics, in_train):\n        self.metrics, self.in_train = listify(metrics), in_train\n\n    def reset(self):\n        self.tot_loss, self.count = 0.0, 0\n        self.tot_mets = [0.0] * len(self.metrics)\n\n    @property\n    def all_stats(self):\n        return [self.tot_loss.item()] + self.tot_mets\n\n    @property\n    def avg_stats(self):\n        return [o / self.count for o in self.all_stats]\n\n    def accumulate(self, run):\n        bn = run.xb.shape[0]\n        self.tot_loss += run.loss * bn\n        self.count += bn\n        for i, m in enumerate(self.metrics):\n            self.tot_mets[i] += m(run.pred, run.yb) * bn\n\n    def __repr__(self):\n        if not self.count:\n            return \"\"\n        return f\"{'train' if self.in_train else 'valid'}: {self.avg_stats}\"\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.AvgStatsCallBack","title":"<code>AvgStatsCallBack</code>","text":"<p>               Bases: <code>Callback</code></p> <p>AvgStatsCallBack class is a custom callback used to track and print average statistics for training and validation phases during the training loop.</p> <p>Arguments:     metrics: A list of metric functions to evaluate during training and validation.</p> <p>Methods:     init: Initializes the callback with given metrics and sets up AvgStats objects for both training and validation phases.     begin_epoch: Resets the statistics at the beginning of each epoch.     after_loss: Accumulates the metrics after computing the loss, differentiating between training and validation phases.     after_epoch: Prints the accumulated statistics for both training and validation phases after each epoch.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class AvgStatsCallBack(Callback):\n    \"\"\"\n    AvgStatsCallBack class is a custom callback used to track and print average statistics for training and validation phases during the training loop.\n\n    Arguments:\n        metrics: A list of metric functions to evaluate during training and validation.\n\n    Methods:\n        __init__: Initializes the callback with given metrics and sets up AvgStats objects for both training and validation phases.\n        begin_epoch: Resets the statistics at the beginning of each epoch.\n        after_loss: Accumulates the metrics after computing the loss, differentiating between training and validation phases.\n        after_epoch: Prints the accumulated statistics for both training and validation phases after each epoch.\n    \"\"\"\n    def __init__(self, metrics):\n        self.train_stats, self.valid_stats = AvgStats(metrics, True), AvgStats(\n            metrics, False\n        )\n\n    def begin_epoch(self):\n        self.train_stats.reset()\n        self.valid_stats.reset()\n\n    def after_loss(self):\n        stats = self.train_stats if self.in_train else self.valid_stats\n        with torch.no_grad():\n            stats.accumulate(self.run)\n\n    def after_epoch(self):\n        print(self.train_stats)\n        print(self.valid_stats)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Callback","title":"<code>Callback</code>","text":"<p>Callback class is a base class designed for handling different callback functions during an event-driven process. It provides functionality to set a runner, retrieve the class name in snake_case format, directly call callback methods, and delegate attribute access to the runner if the attribute does not exist in the Callback class.</p> <p>The _order is used to decide the order of Callbacks.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class Callback:\n    \"\"\"\n\n        Callback class is a base class designed for handling different callback functions during\n        an event-driven process. It provides functionality to set a runner, retrieve the class\n        name in snake_case format, directly call callback methods, and delegate attribute access\n        to the runner if the attribute does not exist in the Callback class.\n\n        The _order is used to decide the order of Callbacks.\n\n    \"\"\"\n    _order = 0\n\n    def set_runner(self, run) -&gt; None:\n        self.run = run\n\n    @property\n    def name(self):\n        name = re.sub(r\"Callback$\", \"\", self.__class__.__name__)\n        return camel2snake(name or \"callback\")\n\n    def __call__(self, cb_name):\n        f = getattr(self, cb_name, None)\n        if f and f():\n            return True\n        return False\n\n    def __getattr__(self, k):\n        return getattr(self.run, k)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler","title":"<code>ParamScheduler</code>","text":"<p>               Bases: <code>Callback</code></p> <p>Manages scheduling of parameter adjustments over the course of training.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class ParamScheduler(Callback):\n    \"\"\"\n    Manages scheduling of parameter adjustments over the course of training.\n    \"\"\"\n    _order = 1\n\n    def __init__(self, pname, sched_funcs):\n        self.pname, self.sched_funcs = pname, sched_funcs\n\n    def begin_fit(self):\n        \"\"\"\n        Prepare the scheduler at the start of the fitting process.\n        This method ensures that sched_funcs is a list with one function per parameter group.\n        \"\"\"\n        if not isinstance(self.sched_funcs, (list, tuple)):\n            self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)\n\n    def set_param(self):\n        \"\"\"\n        Adjust the parameter value for each parameter group based on the scheduling function.\n        Ensures the number of scheduling functions matches the number of parameter groups.\n        \"\"\"\n        assert len(self.opt.param_groups) == len(self.sched_funcs)\n        for pg, f in zip(self.opt.param_groups, self.sched_funcs):\n            pg[self.pname] = f(self.n_epochs / self.epochs)\n\n    def begin_batch(self):\n        \"\"\"\n        Apply parameter adjustments at the beginning of each batch if in training mode.\n        \"\"\"\n        if self.in_train:\n            self.set_param()\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler.begin_batch","title":"<code>begin_batch()</code>","text":"<p>Apply parameter adjustments at the beginning of each batch if in training mode.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def begin_batch(self):\n    \"\"\"\n    Apply parameter adjustments at the beginning of each batch if in training mode.\n    \"\"\"\n    if self.in_train:\n        self.set_param()\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler.begin_fit","title":"<code>begin_fit()</code>","text":"<p>Prepare the scheduler at the start of the fitting process. This method ensures that sched_funcs is a list with one function per parameter group.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def begin_fit(self):\n    \"\"\"\n    Prepare the scheduler at the start of the fitting process.\n    This method ensures that sched_funcs is a list with one function per parameter group.\n    \"\"\"\n    if not isinstance(self.sched_funcs, (list, tuple)):\n        self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler.set_param","title":"<code>set_param()</code>","text":"<p>Adjust the parameter value for each parameter group based on the scheduling function. Ensures the number of scheduling functions matches the number of parameter groups.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def set_param(self):\n    \"\"\"\n    Adjust the parameter value for each parameter group based on the scheduling function.\n    Ensures the number of scheduling functions matches the number of parameter groups.\n    \"\"\"\n    assert len(self.opt.param_groups) == len(self.sched_funcs)\n    for pg, f in zip(self.opt.param_groups, self.sched_funcs):\n        pg[self.pname] = f(self.n_epochs / self.epochs)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder","title":"<code>Recorder</code>","text":"<p>               Bases: <code>Callback</code></p> <p>Recorder is a callback class used to record learning rates and losses during the training process.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class Recorder(Callback):\n    \"\"\"\n        Recorder is a callback class used to record learning rates and losses during the training process.\n    \"\"\"\n    def begin_fit(self):\n        \"\"\"\n        Initializes attributes necessary for the fitting process.\n\n        Sets up learning rates and losses storage.\n\n        Attributes:\n            self.lrs (list): A list of lists, where each inner list will hold learning rates for a parameter group.\n            self.losses (list): An empty list to store loss values during the fitting process.\n        \"\"\"\n        self.lrs = [[] for _ in self.opt.param_groups]\n        self.losses = []\n\n    def after_batch(self):\n        \"\"\"\n        Handles operations to execute after each training batch.\n\n        Modifies the learning rate for each parameter group in the optimizer \n        and appends the current learning rate and loss to the corresponding lists.\n\n        \"\"\"\n        if not self.in_train:\n            return\n        for pg, lr in zip(self.opt.param_groups, self.lrs):\n            lr.append(pg[\"lr\"])\n        self.losses.append(self.loss.detach().cpu())\n\n    def plot_lr(self, pgid=-1):\n        \"\"\"\n        Plots the learning rate for a given parameter group.\n        \"\"\"\n        plt.plot(self.lrs[pgid])\n\n    def plot_loss(self, skip_last=0):\n        \"\"\"\n        Plots the loss for a given parameter group.\n        \"\"\"\n        plt.plot(self.losses[: len(self.losses) - skip_last])\n\n    def plot(self, skip_last=0, pgid=-1):\n        \"\"\"\n        Generates a plot of the loss values against the learning rates.\n        \"\"\"\n        losses = [o.item() for o in self.losses]\n        lrs = self.lrs[pgid]\n        n = len(losses) - skip_last\n        plt.xscale(\"log\")\n        plt.plot(lrs[:n], losses[:n])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.after_batch","title":"<code>after_batch()</code>","text":"<p>Handles operations to execute after each training batch.</p> <p>Modifies the learning rate for each parameter group in the optimizer  and appends the current learning rate and loss to the corresponding lists.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def after_batch(self):\n    \"\"\"\n    Handles operations to execute after each training batch.\n\n    Modifies the learning rate for each parameter group in the optimizer \n    and appends the current learning rate and loss to the corresponding lists.\n\n    \"\"\"\n    if not self.in_train:\n        return\n    for pg, lr in zip(self.opt.param_groups, self.lrs):\n        lr.append(pg[\"lr\"])\n    self.losses.append(self.loss.detach().cpu())\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.begin_fit","title":"<code>begin_fit()</code>","text":"<p>Initializes attributes necessary for the fitting process.</p> <p>Sets up learning rates and losses storage.</p> <p>Attributes:     self.lrs (list): A list of lists, where each inner list will hold learning rates for a parameter group.     self.losses (list): An empty list to store loss values during the fitting process.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def begin_fit(self):\n    \"\"\"\n    Initializes attributes necessary for the fitting process.\n\n    Sets up learning rates and losses storage.\n\n    Attributes:\n        self.lrs (list): A list of lists, where each inner list will hold learning rates for a parameter group.\n        self.losses (list): An empty list to store loss values during the fitting process.\n    \"\"\"\n    self.lrs = [[] for _ in self.opt.param_groups]\n    self.losses = []\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.plot","title":"<code>plot(skip_last=0, pgid=-1)</code>","text":"<p>Generates a plot of the loss values against the learning rates.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def plot(self, skip_last=0, pgid=-1):\n    \"\"\"\n    Generates a plot of the loss values against the learning rates.\n    \"\"\"\n    losses = [o.item() for o in self.losses]\n    lrs = self.lrs[pgid]\n    n = len(losses) - skip_last\n    plt.xscale(\"log\")\n    plt.plot(lrs[:n], losses[:n])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.plot_loss","title":"<code>plot_loss(skip_last=0)</code>","text":"<p>Plots the loss for a given parameter group.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def plot_loss(self, skip_last=0):\n    \"\"\"\n    Plots the loss for a given parameter group.\n    \"\"\"\n    plt.plot(self.losses[: len(self.losses) - skip_last])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.plot_lr","title":"<code>plot_lr(pgid=-1)</code>","text":"<p>Plots the learning rate for a given parameter group.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def plot_lr(self, pgid=-1):\n    \"\"\"\n    Plots the learning rate for a given parameter group.\n    \"\"\"\n    plt.plot(self.lrs[pgid])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.TrainEvalCallback","title":"<code>TrainEvalCallback</code>","text":"<p>               Bases: <code>Callback</code></p> <p>TrainEvalCallback class is a custom callback used during the training and validation phases of a machine learning model to perform specific actions at the beginning and after certain events.</p> <p>Methods:</p> <p>begin_fit():     Initialize the number of epochs and iteration counts at the start     of the fitting process.</p> <p>after_batch():     Update the epoch and iteration counts after each batch during     training.</p> <p>begin_epoch():     Set the current epoch, switch the model to training mode, and     indicate that the model is in training.</p> <p>begin_validate():     Switch the model to evaluation mode and indicate that the model     is in validation.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class TrainEvalCallback(Callback):\n    \"\"\"\n        TrainEvalCallback class is a custom callback used during the training\n        and validation phases of a machine learning model to perform specific\n        actions at the beginning and after certain events.\n\n        Methods:\n\n        begin_fit():\n            Initialize the number of epochs and iteration counts at the start\n            of the fitting process.\n\n        after_batch():\n            Update the epoch and iteration counts after each batch during\n            training.\n\n        begin_epoch():\n            Set the current epoch, switch the model to training mode, and\n            indicate that the model is in training.\n\n        begin_validate():\n            Switch the model to evaluation mode and indicate that the model\n            is in validation.\n    \"\"\"\n    def begin_fit(self):\n        self.run.n_epochs = 0\n        self.run.n_iter = 0\n\n    def after_batch(self):\n        if not self.in_train:\n            return\n        self.run.n_epochs += 1.0 / self.iters\n        self.run.n_iter += 1\n\n    def begin_epoch(self):\n        self.run.n_epochs = self.epoch\n        self.model.train()\n        self.run.in_train = True\n\n    def begin_validate(self):\n        self.model.eval()\n        self.run.in_train = False\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.annealer","title":"<code>annealer(f)</code>","text":"<p>A decorator function for creating a partially applied function with predefined start and end arguments. The inner function <code>_inner</code> captures the <code>start</code> and <code>end</code> parameters and returns a <code>partial</code> object that fixes these parameters for the decorated function <code>f</code>.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def annealer(f) -&gt; callable:\n    \"\"\"\n    A decorator function for creating a partially applied function with predefined start and end arguments.\n    The inner function `_inner` captures the `start` and `end` parameters and returns a `partial` object that fixes these parameters for the decorated function `f`.\n    \"\"\"\n    def _inner(start, end):\n        return partial(f, start, end)\n\n    return _inner\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.camel2snake","title":"<code>camel2snake(name)</code>","text":"<p>Convert <code>name</code> from camel case to snake case.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def camel2snake(name : str) -&gt; str:\n    \"\"\"\n    Convert `name` from camel case to snake case.\n    \"\"\"\n    s1 = re.sub(_camel_re1, r\"\\1_\\2\", name)\n    return re.sub(_camel_re2, r\"\\1_\\2\", s1).lower()\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.combine_scheds","title":"<code>combine_scheds(pcts, scheds)</code>","text":"<p>Combine multiple scheduling functions.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def combine_scheds(pcts: Iterable[float], scheds: Iterable[callable]) -&gt; callable:\n    \"\"\"\n    Combine multiple scheduling functions.\n    \"\"\"\n    assert sum(pcts) == 1.0\n    pcts = torch.tensor([0] + listify(pcts))\n    assert torch.all(pcts &gt;= 0)\n    pcts = torch.cumsum(pcts, 0)\n\n    def _inner(pos):\n        idx = (pos &gt;= pcts).nonzero().max()\n        actual_pos = (pos - pcts[idx]) / (pcts[idx + 1] - pcts[idx])\n        return scheds[idx](actual_pos)\n\n    return _inner\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.listify","title":"<code>listify(o=None)</code>","text":"<p>Convert <code>o</code> to list. If <code>o</code> is None, return empty list.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def listify(o = None) -&gt; list:\n    \"\"\"\n    Convert `o` to list. If `o` is None, return empty list.\n    \"\"\"\n    if o is None:\n        return []\n    if isinstance(o, list):\n        return o\n    if isinstance(o, str):\n        return [o]\n    if isinstance(o, Iterable):\n        return list(o)\n    return [o]\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_cos","title":"<code>sched_cos(start, end, pos)</code>","text":"<p>A cosine schedule function.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_cos(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    A cosine schedule function.\n    \"\"\"\n    return start + (1 + math.cos(math.pi * (1 - pos))) * (end - start) / 2\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_exp","title":"<code>sched_exp(start, end, pos)</code>","text":"<p>Exponential schedule function.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_exp(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    Exponential schedule function.\n    \"\"\"\n    return start * (end / start) ** pos\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_lin","title":"<code>sched_lin(start, end, pos)</code>","text":"<p>A linear schedule function.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_lin(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    A linear schedule function.\n    \"\"\"\n    return start + pos * (end - start)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_no","title":"<code>sched_no(start, end, pos)</code>","text":"<p>Disabled scheduling.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_no(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    Disabled scheduling.\n    \"\"\"\n    return start\n</code></pre>"},{"location":"pytorch/API%20reference/Custom%20Datasets/","title":"Custom Datasets","text":"<p>This module contains custom dataset classes for handling image and tabular data from OpenML in PyTorch. To add support for new data types, new classes can be added to this module.</p>"},{"location":"pytorch/API%20reference/Custom%20Datasets/#custom_datasets.OpenMLImageDataset","title":"<code>OpenMLImageDataset</code>","text":"<p>               Bases: <code>Dataset</code></p> <p>Class representing an image dataset from OpenML for use in PyTorch.</p> <p>Methods:</p> <pre><code>__init__(self, X, y, image_size, image_dir, transform_x=None, transform_y=None)\n    Initializes the dataset with given data, image size, directory, and optional transformations.\n\n__getitem__(self, idx)\n    Retrieves an image and its corresponding label (if available) from the dataset at the specified index. Applies transformations if provided.\n\n__len__(self)\n    Returns the total number of images in the dataset.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/custom_datasets.py</code> <pre><code>class OpenMLImageDataset(Dataset):\n    \"\"\"\n        Class representing an image dataset from OpenML for use in PyTorch.\n\n        Methods:\n\n            __init__(self, X, y, image_size, image_dir, transform_x=None, transform_y=None)\n                Initializes the dataset with given data, image size, directory, and optional transformations.\n\n            __getitem__(self, idx)\n                Retrieves an image and its corresponding label (if available) from the dataset at the specified index. Applies transformations if provided.\n\n            __len__(self)\n                Returns the total number of images in the dataset.\n    \"\"\"\n    def __init__(self, X, y, image_size, image_dir, transform_x = None, transform_y = None):\n        self.X = X\n        self.y = y\n        self.image_size = image_size\n        self.image_dir = image_dir\n        self.transform_x = transform_x\n        self.transform_y = transform_y\n\n    def __getitem__(self, idx):\n        img_name = str(os.path.join(self.image_dir, self.X.iloc[idx, 0]))\n        image = read_image(img_name)\n        image = image.float()\n        image = T.Resize((self.image_size, self.image_size))(image)\n        if self.transform_x is not None:\n            image = self.transform_x(image)\n        if self.y is not None:\n            label = self.y.iloc[idx]\n            if label is not None:\n                if self.transform_y is not None:\n                    label = self.transform_y(label)\n                return image, label\n        else:\n            return image\n\n    def __len__(self):\n        return len(self.X)\n</code></pre>"},{"location":"pytorch/API%20reference/Custom%20Datasets/#custom_datasets.OpenMLTabularDataset","title":"<code>OpenMLTabularDataset</code>","text":"<p>               Bases: <code>Dataset</code></p> <p>OpenMLTabularDataset</p> <p>A custom dataset class to handle tabular data from OpenML (or any similar tabular dataset). It encodes categorical features and the target column using LabelEncoder from sklearn.</p> <p>Methods:     init(X, y) : Initializes the dataset with the data and the target column.                      Encodes the categorical features and target if provided.</p> <pre><code>__getitem__(idx): Retrieves the input data and target value at the specified index.\n                  Converts the data to tensors and returns them.\n\n__len__(): Returns the length of the dataset.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/custom_datasets.py</code> <pre><code>class OpenMLTabularDataset(Dataset):\n    \"\"\"\n    OpenMLTabularDataset\n\n    A custom dataset class to handle tabular data from OpenML (or any similar tabular dataset).\n    It encodes categorical features and the target column using LabelEncoder from sklearn.\n\n    Methods:\n        __init__(X, y) : Initializes the dataset with the data and the target column.\n                         Encodes the categorical features and target if provided.\n\n        __getitem__(idx): Retrieves the input data and target value at the specified index.\n                          Converts the data to tensors and returns them.\n\n        __len__(): Returns the length of the dataset.\n    \"\"\"\n    def __init__(self, X, y):\n        self.data = X\n        # self.target_col_name = target_col\n        for col in self.data.select_dtypes(include=['object', 'category']):\n            # convert to float\n            self.data[col] = self.data[col].astype('category').cat.codes\n        self.label_mapping = None\n\n        # self.label_mapping = preprocessing.LabelEncoder()\n        # try:\n        #     self.data = self.data.apply(self.label_mapping.fit_transform)\n        # except ValueError:\n        #     pass\n\n        # try:\n        #     self.y = self.label_mapping.fit_transform(y)\n        # except ValueError:\n        #     self.y = None\n        self.y = y\n\n    def __getitem__(self, idx):\n        # x is the input data, y is the target value from the target column\n        x = self.data.iloc[idx, :]\n        x = torch.tensor(x.values.astype('float32'))\n        if self.y is not None:\n            y = self.y[idx]\n            y = torch.tensor(y)\n            return x, y\n        else:\n            return x\n\n\n    def __len__(self):\n        return len(self.data)\n</code></pre>"},{"location":"pytorch/API%20reference/Metrics/","title":"Metrics","text":"<p>This module provides utility functions for evaluating model performance and activation functions. It includes functions to compute the accuracy, top-k accuracy of model predictions, and the sigmoid function.</p>"},{"location":"pytorch/API%20reference/Metrics/#metrics.accuracy","title":"<code>accuracy(out, yb)</code>","text":"<p>Computes the accuracy of model predictions.</p> <p>Parameters: out (Tensor): The output tensor from the model, containing predicted class scores. yb (Tensor): The ground truth labels tensor.</p> <p>Returns: Tensor: The mean accuracy of the predictions, computed as a float tensor.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/metrics.py</code> <pre><code>def accuracy(out, yb):\n    \"\"\"\n\n    Computes the accuracy of model predictions.\n\n    Parameters:\n    out (Tensor): The output tensor from the model, containing predicted class scores.\n    yb (Tensor): The ground truth labels tensor.\n\n    Returns:\n    Tensor: The mean accuracy of the predictions, computed as a float tensor.\n    \"\"\"\n    return (torch.argmax(out, dim=1) == yb.long()).float().mean()\n</code></pre>"},{"location":"pytorch/API%20reference/Metrics/#metrics.accuracy_topk","title":"<code>accuracy_topk(out, yb, k=5)</code>","text":"<p>Computes the top-k accuracy of the given model outputs.</p> <p>Args:     out (torch.Tensor): The output predictions of the model, of shape (batch_size, num_classes).     yb (torch.Tensor): The ground truth labels, of shape (batch_size,).     k (int, optional): The number of top predictions to consider. Default is 5.</p> <p>Returns:     float: The top-k accuracy as a float value.</p> <p>The function calculates how often the true label is among the top-k predicted labels.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/metrics.py</code> <pre><code>def accuracy_topk(out, yb, k=5):\n    \"\"\"\n\n    Computes the top-k accuracy of the given model outputs.\n\n    Args:\n        out (torch.Tensor): The output predictions of the model, of shape (batch_size, num_classes).\n        yb (torch.Tensor): The ground truth labels, of shape (batch_size,).\n        k (int, optional): The number of top predictions to consider. Default is 5.\n\n    Returns:\n        float: The top-k accuracy as a float value.\n\n    The function calculates how often the true label is among the top-k predicted labels.\n    \"\"\"\n    return (torch.topk(out, k, dim=1)[1] == yb.long().unsqueeze(1)).float().mean()\n</code></pre>"},{"location":"pytorch/API%20reference/Metrics/#metrics.sigmoid","title":"<code>sigmoid(x)</code>","text":"<p>Computes the sigmoid function</p> <p>The sigmoid function is defined as 1 / (1 + exp(-x)). This function is used to map any real-valued number into the range (0, 1). It is widely used in machine learning, especially in logistic regression and neural networks.</p> <p>Args:     x (numpy.ndarray or float): The input value or array over which the     sigmoid function should be applied.</p> <p>Returns:     numpy.ndarray or float: The sigmoid of the input value or array.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/metrics.py</code> <pre><code>def sigmoid(x):\n    \"\"\"\n    Computes the sigmoid function\n\n    The sigmoid function is defined as 1 / (1 + exp(-x)). This function is used\n    to map any real-valued number into the range (0, 1). It is widely used in\n    machine learning, especially in logistic regression and neural networks.\n\n    Args:\n        x (numpy.ndarray or float): The input value or array over which the\n        sigmoid function should be applied.\n\n    Returns:\n        numpy.ndarray or float: The sigmoid of the input value or array.\n    \"\"\"\n    return 1 / (1 + np.exp(-x))\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/","title":"OpenML Connection","text":"<p>This module defines the Pytorch extension for OpenML-python.</p>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension","title":"<code>PytorchExtension</code>","text":"<p>               Bases: <code>Extension</code></p> <p>Connect Pytorch to OpenML-Python.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>class PytorchExtension(Extension):\n    \"\"\"Connect Pytorch to OpenML-Python.\"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    def can_handle_flow(cls, flow: 'OpenMLFlow') -&gt; bool:\n        \"\"\"Check whether a given describes a Pytorch estimator.\n\n        This is done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return cls._is_pytorch_flow(flow)\n\n    @classmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model is an instance of ``torch.nn.Module``.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        from torch.nn import Module\n        return isinstance(model, Module)\n\n    ################################################################################################\n    # Method for dataloader \n\n\n\n    ################################################################################################\n    # Methods for flow serialization and de-serialization\n\n    def flow_to_model(self, flow: 'OpenMLFlow', initialize_with_defaults: bool = False) -&gt; Any:\n        \"\"\"Initializes a Pytorch model based on a flow.\n\n        Parameters\n        ----------\n        flow : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        return self._deserialize_pytorch(flow, initialize_with_defaults=initialize_with_defaults)\n\n    def _deserialize_pytorch(\n        self,\n        o: Any,\n        components: Optional[Dict] = None,\n        initialize_with_defaults: bool = False,\n        recursion_depth: int = 0,\n    ) -&gt; Any:\n        \"\"\"Recursive function to deserialize a Pytorch flow.\n\n        This function delegates all work to the respective functions to deserialize special data\n        structures etc.\n\n        Parameters\n        ----------\n        o : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        components : dict\n\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        recursion_depth : int\n            The depth at which this flow is called, mostly for debugging\n            purposes\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n\n        logging.info('-%s flow_to_pytorch START o=%s, components=%s, '\n                     'init_defaults=%s' % ('-' * recursion_depth, o, components,\n                                           initialize_with_defaults))\n        depth_pp = recursion_depth + 1  # shortcut var, depth plus plus\n\n        # First, we need to check whether the presented object is a json string.\n        # JSON strings are used to encoder parameter values. By passing around\n        # json strings for parameters, we make sure that we can flow_to_pytorch\n        # the parameter values to the correct type.\n\n        if isinstance(o, str):\n            try:\n                o = json.loads(o)\n            except JSONDecodeError:\n                pass\n\n        if isinstance(o, dict):\n            # Check if the dict encodes a 'special' object, which could not\n            # easily converted into a string, but rather the information to\n            # re-create the object were stored in a dictionary.\n            if 'oml-python:serialized_object' in o:\n                serialized_type = o['oml-python:serialized_object']\n                value = o['value']\n                if serialized_type == 'type':\n                    rval = self._deserialize_type(value)\n                elif serialized_type == 'function':\n                    rval = self._deserialize_function(value)\n                elif serialized_type == 'methoddescriptor':\n                    rval = self._deserialize_methoddescriptor(value)\n                elif serialized_type == 'component_reference':\n                    assert components is not None  # Necessary for mypy\n                    value = self._deserialize_pytorch(value, recursion_depth=depth_pp)\n                    step_name = value['step_name']\n                    key = value['key']\n                    if key not in components:\n                        key = str(key)\n                    component = self._deserialize_pytorch(\n                        components[key],\n                        initialize_with_defaults=initialize_with_defaults,\n                        recursion_depth=depth_pp\n                    )\n                    # The component is now added to where it should be used\n                    # later. It should not be passed to the constructor of the\n                    # main flow object.\n                    del components[key]\n                    if step_name is None:\n                        rval = component\n                    elif 'argument_1' not in value:\n                        rval = (step_name, component)\n                    else:\n                        rval = (step_name, component, value['argument_1'])\n                else:\n                    raise ValueError('Cannot flow_to_pytorch %s' % serialized_type)\n\n            else:\n                rval = OrderedDict(\n                    (\n                        self._deserialize_pytorch(\n                            o=key,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                        ),\n                        self._deserialize_pytorch(\n                            o=value,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                        )\n                    )\n                    for key, value in sorted(o.items())\n                )\n        elif isinstance(o, (list, tuple)):\n            rval = [\n                self._deserialize_pytorch(\n                    o=element,\n                    components=components,\n                    initialize_with_defaults=initialize_with_defaults,\n                    recursion_depth=depth_pp,\n                )\n                for element in o\n            ]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, (bool, int, float, str)) or o is None:\n            rval = o\n        elif isinstance(o, OpenMLFlow):\n            if not self._is_pytorch_flow(o):\n                raise ValueError('Only pytorch flows can be reinstantiated')\n            rval = self._deserialize_model(\n                flow=o,\n                keep_defaults=initialize_with_defaults,\n                recursion_depth=recursion_depth,\n            )\n        else:\n            raise TypeError(o)\n        logging.info('-%s flow_to_pytorch END   o=%s, rval=%s'\n                     % ('-' * recursion_depth, o, rval))\n        return rval\n\n    def model_to_flow(self, model: Any, custom_name: Optional[str] = None) -&gt; 'OpenMLFlow':\n        \"\"\"Transform a Pytorch model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n        # Necessary to make pypy not complain about all the different possible return types\n        return self._serialize_pytorch(model, custom_name)\n\n    def _serialize_pytorch(self, o: Any, parent_model: Optional[Any] = None, custom_name: Optional[str] = None) -&gt; Any:\n        rval = None  # type: Any\n        if self.is_estimator(o):\n            # is the main model or a submodel\n            rval = self._serialize_model(o, custom_name)\n        elif isinstance(o, (list, tuple)):\n            rval = [self._serialize_pytorch(element, parent_model) for element in o]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, SIMPLE_TYPES) or o is None:\n            if isinstance(o, tuple(SIMPLE_NUMPY_TYPES)):\n                o = o.item()\n            # base parameter values\n            rval = o\n        elif isinstance(o, dict):\n            if not isinstance(o, OrderedDict):\n                o = OrderedDict([(key, value) for key, value in sorted(o.items())])\n\n            rval = OrderedDict()\n            for key, value in o.items():\n                if not isinstance(key, str):\n                    raise TypeError('Can only use string as keys, you passed '\n                                    'type %s for value %s.' %\n                                    (type(key), str(key)))\n                key = self._serialize_pytorch(key, parent_model)\n                value = self._serialize_pytorch(value, parent_model)\n                rval[key] = value\n            rval = rval\n        elif isinstance(o, type):\n            rval = self._serialize_type(o)\n        # This only works for user-defined functions (and not even partial).\n        # I think this is exactly what we want here as there shouldn't be any\n        # built-in or functool.partials in a pipeline\n        elif inspect.isfunction(o):\n            rval = self._serialize_function(o)\n        elif inspect.ismethoddescriptor(o):\n            rval = self._serialize_methoddescriptor(o)\n        else:\n            raise TypeError(o, type(o))\n        return rval\n\n    def get_version_information(self) -&gt; List[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Libraries listed are ``Python``, ``pytorch``, ``numpy`` and ``scipy``.\n\n        Returns\n        -------\n        List\n        \"\"\"\n\n        # This can possibly be done by a package such as pyxb, but I could not get\n        # it to work properly.\n        import scipy\n        import numpy\n\n        major, minor, micro, _, _ = sys.version_info\n        python_version = 'Python_{}.'.format(\n            \".\".join([str(major), str(minor), str(micro)]))\n        pytorch_version = 'Torch_{}.'.format(torch.__version__)\n        numpy_version = 'NumPy_{}.'.format(numpy.__version__)\n        scipy_version = 'SciPy_{}.'.format(scipy.__version__)\n        pytorch_version_formatted = pytorch_version.replace('+','_')\n        return [python_version, pytorch_version_formatted, numpy_version, scipy_version]\n\n    def create_setup_string(self, model: Any) -&gt; str:\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n        run_environment = \" \".join(self.get_version_information())\n        return run_environment + \" \" + str(model)\n\n    @classmethod\n    def _is_pytorch_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        return (\n            flow.external_version.startswith('torch==')\n            or ',torch==' in flow.external_version\n        )\n\n    def _serialize_model(self, model: Any, custom_name: Optional[str] = None) -&gt; OpenMLFlow:\n        \"\"\"Create an OpenMLFlow.\n\n        Calls `pytorch_to_flow` recursively to properly serialize the\n        parameters to strings and the components (other models) to OpenMLFlows.\n\n        Parameters\n        ----------\n        model : pytorch estimator\n\n        Returns\n        -------\n        OpenMLFlow\n\n        \"\"\"\n\n        # Get all necessary information about the model objects itself\n        parameters, parameters_meta_info, subcomponents, subcomponents_explicit = \\\n            self._extract_information_from_model(model)\n\n        # Check that a component does not occur multiple times in a flow as this\n        # is not supported by OpenML\n        self._check_multiple_occurence_of_component_in_flow(model, subcomponents)\n\n        import zlib\n        import os\n\n        # class_name = model.__module__ + \".\" + model.__class__.__name__\n        class_name = 'torch.nn' + \".\" + model.__class__.__name__\n        class_name += '.'\n        class_name += format(zlib.crc32(bytearray(os.urandom(32))), 'x')\n        class_name += format(zlib.crc32(bytearray(os.urandom(32))), 'x')\n\n        name = class_name\n\n        # Get the external versions of all sub-components\n        external_version = self._get_external_version_string(model, subcomponents)\n\n        dependencies = '\\n'.join([\n            self._format_external_version(\n                'torch',\n                torch.__version__,\n            ),\n            'numpy&gt;=1.6.1',\n            'scipy&gt;=0.9',\n        ])\n\n        torch_version = self._format_external_version('torch', torch.__version__)\n        torch_version_formatted = torch_version.replace('==', '_')\n        torch_version_formatted = torch_version_formatted.replace('+', '_')\n\n        flow = OpenMLFlow(name=name,\n                          class_name=class_name,\n                          description='Automatically created pytorch flow.',\n                          model=model,\n                          components=subcomponents,\n                          parameters=parameters,\n                          parameters_meta_info=parameters_meta_info,\n                          external_version=external_version,\n                          tags=['openml-python', 'pytorch',\n                                'python', torch_version_formatted],\n                          language='English',\n                          dependencies=dependencies, \n                          custom_name=custom_name)\n\n        return flow\n\n    def _get_external_version_string(\n        self,\n        model: Any,\n        sub_components: Dict[str, OpenMLFlow],\n    ) -&gt; str:\n        # Create external version string for a flow, given the model and the\n        # already parsed dictionary of sub_components. Retrieves the external\n        # version of all subcomponents, which themselves already contain all\n        # requirements for their subcomponents. The external version string is a\n        # sorted concatenation of all modules which are present in this run.\n        model_package_name = model.__module__.split('.')[0]\n        module = importlib.import_module(model_package_name)\n        model_package_version_number = 'module.__version__'  # type: ignore\n        external_version = self._format_external_version(\n            model_package_name, model_package_version_number,\n        )\n        openml_version = self._format_external_version('openml', openml.__version__)\n        torch_version = self._format_external_version('torch', torch.__version__)\n        external_versions = set()\n        external_versions.add(external_version)\n        external_versions.add(openml_version)\n        external_versions.add(torch_version)\n        for visitee in sub_components.values():\n            for external_version in visitee.external_version.split(','):\n                external_versions.add(external_version)\n        return ','.join(list(sorted(external_versions)))\n\n    def _check_multiple_occurence_of_component_in_flow(\n        self,\n        model: Any,\n        sub_components: Dict[str, OpenMLFlow],\n    ) -&gt; None:\n        to_visit_stack = []  # type: List[OpenMLFlow]\n        to_visit_stack.extend(sub_components.values())\n        known_sub_components = set()  # type: Set[str]\n        while len(to_visit_stack) &gt; 0:\n            visitee = to_visit_stack.pop()\n            if visitee.name in known_sub_components:\n                raise ValueError('Found a second occurence of component %s when '\n                                 'trying to serialize %s.' % (visitee.name, model))\n            else:\n                known_sub_components.add(visitee.name)\n                to_visit_stack.extend(visitee.components.values())\n\n    def _is_container_module(self, module: torch.nn.Module) -&gt; bool:\n        if isinstance(module,\n                      (torch.nn.Sequential,\n                       torch.nn.ModuleDict,\n                       torch.nn.ModuleList)):\n            return True\n        if module in (torch.nn.modules.container.Sequential,\n                      torch.nn.modules.container.ModuleDict,\n                      torch.nn.modules.container.ModuleList):\n            return True\n        return False\n\n    def _get_module_hyperparameters(self, module: torch.nn.Module,\n                                    parameters: Dict[str, torch.nn.Parameter]) -&gt; Dict[str, Any]:\n        # Extract the signature of the module constructor\n        main_signature = inspect.signature(module.__init__)\n        params = dict()  # type: Dict[str, Any]\n\n        check_bases = False  # type: bool\n        for param_name, param in main_signature.parameters.items():\n            # Skip hyper-parameters which are actually parameters.\n            if param_name in parameters.keys():\n                continue\n\n            # Skip *args and **kwargs, and check the base classes instead.\n            if param.kind in (inspect.Parameter.VAR_POSITIONAL,\n                              inspect.Parameter.VAR_KEYWORD):\n                check_bases = True\n                continue\n\n            # Extract the hyperparameter from the module.\n            if hasattr(module, param_name):\n                params[param_name] = getattr(module, param_name)\n\n        if check_bases:\n            for base in module.__class__.__bases__:\n                # Extract the signature  of the base constructor\n                base_signature = inspect.signature(base.__init__)\n\n                for param_name, param in base_signature.parameters.items():\n                    # Skip hyper-parameters which are actually parameters.\n                    if param_name in parameters.keys():\n                        continue\n\n                    # Skip *args and **kwargs since they are not relevant.\n                    if param.kind in (inspect.Parameter.VAR_POSITIONAL,\n                                      inspect.Parameter.VAR_KEYWORD):\n                        continue\n\n                    # Extract the hyperparameter from the module.\n                    if hasattr(module, param_name):\n                        params[param_name] = getattr(module, param_name)\n\n        from .layers import Functional\n        if isinstance(module, Functional):\n            params['args'] = getattr(module, 'args')\n            params['kwargs'] = getattr(module, 'kwargs')\n\n        return params\n\n    def _get_module_descriptors(self, model: torch.nn.Module, deep=True) -&gt; Dict[str, Any]:\n        # The named children (modules) of the given module.\n        named_children = list((k, v) for (k, v) in model.named_children())\n        # The parameters of the given module and its submodules.\n        model_parameters = dict((k, v) for (k, v) in model.named_parameters())\n\n        parameters = dict()  # type: Dict[str, Any]\n\n        if not self._is_container_module(model):\n            # For non-containers, we simply extract the hyperparameters.\n            parameters = self._get_module_hyperparameters(model, model_parameters)\n        else:\n            # Otherwise we serialize their children as lists of pairs in order\n            # to maintain the order of the sub modules.\n            parameters['children'] = named_children\n\n        # If a deep description is required, append the children to the dictionary of\n        # returned parameters.\n        if deep:\n            named_children_dict = dict(named_children)\n            parameters = {**parameters, **named_children_dict}\n\n        return parameters\n\n    def _extract_information_from_model(\n        self,\n        model: Any,\n    ) -&gt; Tuple[\n        'OrderedDict[str, Optional[str]]',\n        'OrderedDict[str, Optional[Dict]]',\n        'OrderedDict[str, OpenMLFlow]',\n        Set,\n    ]:\n        # This function contains four \"global\" states and is quite long and\n        # complicated. If it gets to complicated to ensure it's correctness,\n        # it would be best to make it a class with the four \"global\" states being\n        # the class attributes and the if/elif/else in the for-loop calls to\n        # separate class methods\n\n        # stores all entities that should become subcomponents\n        sub_components = OrderedDict()  # type: OrderedDict[str, OpenMLFlow]\n        # stores the keys of all subcomponents that should become\n        sub_components_explicit = set()\n        parameters = OrderedDict()  # type: OrderedDict[str, Optional[str]]\n        parameters_meta_info = OrderedDict()  # type: OrderedDict[str, Optional[Dict]]\n\n        model_parameters = self._get_module_descriptors(model, deep=True)\n        for k, v in sorted(model_parameters.items(), key=lambda t: t[0]):\n            rval = self._serialize_pytorch(v, model)\n\n            def flatten_all(list_):\n                \"\"\" Flattens arbitrary depth lists of lists (e.g. [[1,2],[3,[1]]] -&gt; [1,2,3,1]). \"\"\"\n                for el in list_:\n                    if isinstance(el, (list, tuple)):\n                        yield from flatten_all(el)\n                    else:\n                        yield el\n\n            is_non_empty_list_of_lists_with_same_type = (\n                isinstance(rval, (list, tuple))\n                and len(rval) &gt; 0\n                and isinstance(rval[0], (list, tuple))\n                and all([isinstance(rval_i, type(rval[0])) for rval_i in rval])\n            )\n\n            # Check that all list elements are of simple types.\n            nested_list_of_simple_types = (\n                is_non_empty_list_of_lists_with_same_type\n                and all([isinstance(el, SIMPLE_TYPES) for el in flatten_all(rval)])\n            )\n\n            if is_non_empty_list_of_lists_with_same_type and not nested_list_of_simple_types:\n                # If a list of lists is identified that include 'non-simple' types (e.g. objects),\n                # we assume they are steps in a pipeline, feature union, or base classifiers in\n                # a voting classifier.\n                parameter_value = list()  # type: List\n                reserved_keywords = set(self._get_module_descriptors(model, deep=False).keys())\n\n                for sub_component_tuple in rval:\n                    identifier = sub_component_tuple[0]\n                    sub_component = sub_component_tuple[1]\n                    sub_component_type = type(sub_component_tuple)\n                    if not 2 &lt;= len(sub_component_tuple) &lt;= 3:\n                        msg = 'Length of tuple does not match assumptions'\n                        raise ValueError(msg)\n                    if not isinstance(sub_component, (OpenMLFlow, type(None))):\n                        msg = 'Second item of tuple does not match assumptions. ' \\\n                              'Expected OpenMLFlow, got %s' % type(sub_component)\n                        raise TypeError(msg)\n\n                    if identifier in reserved_keywords:\n                        parent_model = \"{}.{}\".format(model.__module__,\n                                                      model.__class__.__name__)\n                        msg = 'Found element shadowing official ' \\\n                              'parameter for %s: %s' % (parent_model,\n                                                        identifier)\n                        raise PyOpenMLError(msg)\n\n                    if sub_component is None:\n                        # In a FeatureUnion it is legal to have a None step\n\n                        pv = [identifier, None]\n                        if sub_component_type is tuple:\n                            parameter_value.append(tuple(pv))\n                        else:\n                            parameter_value.append(pv)\n\n                    else:\n                        # Add the component to the list of components, add a\n                        # component reference as a placeholder to the list of\n                        # parameters, which will be replaced by the real component\n                        # when deserializing the parameter\n                        sub_components_explicit.add(identifier)\n                        sub_components[identifier] = sub_component\n                        component_reference = OrderedDict()  # type: Dict[str, Union[str, Dict]]\n                        component_reference['oml-python:serialized_object'] = 'component_reference'\n                        cr_value = OrderedDict()  # type: Dict[str, Any]\n                        cr_value['key'] = identifier\n                        cr_value['step_name'] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value['argument_1'] = sub_component_tuple[2]\n                        component_reference['value'] = cr_value\n                        parameter_value.append(component_reference)\n\n                # Here (and in the elif and else branch below) are the only\n                # places where we encode a value as json to make sure that all\n                # parameter values still have the same type after\n                # deserialization\n\n                if isinstance(rval, tuple):\n                    parameter_json = json.dumps(tuple(parameter_value))\n                else:\n                    parameter_json = json.dumps(parameter_value)\n                parameters[k] = parameter_json\n\n            elif isinstance(rval, OpenMLFlow):\n\n                # A subcomponent, for example the layers in a sequential model\n                sub_components[k] = rval\n                sub_components_explicit.add(k)\n                component_reference = OrderedDict()\n                component_reference['oml-python:serialized_object'] = 'component_reference'\n                cr_value = OrderedDict()\n                cr_value['key'] = k\n                cr_value['step_name'] = None\n                component_reference['value'] = cr_value\n                cr = self._serialize_pytorch(component_reference, model)\n                parameters[k] = json.dumps(cr)\n\n            else:\n                # a regular hyperparameter\n                rval = json.dumps(rval)\n                parameters[k] = rval\n\n            parameters_meta_info[k] = OrderedDict((('description', None), ('data_type', None)))\n\n        return parameters, parameters_meta_info, sub_components, sub_components_explicit\n\n    def _get_fn_arguments_with_defaults(self, fn_name: Callable) -&gt; Tuple[Dict, Set]:\n        \"\"\"\n        Returns:\n            i) a dict with all parameter names that have a default value, and\n            ii) a set with all parameter names that do not have a default\n\n        Parameters\n        ----------\n        fn_name : callable\n            The function of which we want to obtain the defaults\n\n        Returns\n        -------\n        params_with_defaults: dict\n            a dict mapping parameter name to the default value\n        params_without_defaults: set\n            a set with all parameters that do not have a default value\n        \"\"\"\n        # parameters with defaults are optional, all others are required.\n        signature = inspect.getfullargspec(fn_name)\n        if signature.defaults:\n            optional_params = dict(zip(reversed(signature.args), reversed(signature.defaults)))\n        else:\n            optional_params = dict()\n        required_params = {arg for arg in signature.args if arg not in optional_params}\n        return optional_params, required_params\n\n    def _deserialize_model(\n        self,\n        flow: OpenMLFlow,\n        keep_defaults: bool,\n        recursion_depth: int,\n    ) -&gt; Any:\n        logging.info('-%s deserialize %s' % ('-' * recursion_depth, flow.name))\n        model_name = flow.class_name\n        self._check_dependencies(flow.dependencies)\n\n        parameters = flow.parameters\n        components = flow.components\n        parameter_dict = OrderedDict()  # type: Dict[str, Any]\n\n        # Do a shallow copy of the components dictionary so we can remove the\n        # components from this copy once we added them into the pipeline. This\n        # allows us to not consider them any more when looping over the\n        # components, but keeping the dictionary of components untouched in the\n        # original components dictionary.\n        components_ = copy.copy(components)\n\n        for name in parameters:\n            value = parameters.get(name)\n            logging.info('--%s flow_parameter=%s, value=%s' %\n                         ('-' * recursion_depth, name, value))\n            rval = self._deserialize_pytorch(\n                value,\n                components=components_,\n                initialize_with_defaults=keep_defaults,\n                recursion_depth=recursion_depth + 1,\n            )\n            parameter_dict[name] = rval\n\n        for name in components:\n            if name in parameter_dict:\n                continue\n            if name not in components_:\n                continue\n            value = components[name]\n            logging.info('--%s flow_component=%s, value=%s'\n                         % ('-' * recursion_depth, name, value))\n            rval = self._deserialize_pytorch(\n                value,\n                recursion_depth=recursion_depth + 1,\n            )\n            parameter_dict[name] = rval\n\n        # Remove the unique identifier\n        model_name = model_name.rsplit('.', 1)[0]\n\n        module_name = model_name.rsplit('.', 1)\n        model_class = getattr(importlib.import_module(module_name[0]),\n                              module_name[1])\n\n        if keep_defaults:\n            # obtain all params with a default\n            param_defaults, _ = \\\n                self._get_fn_arguments_with_defaults(model_class.__init__)\n\n            # delete the params that have a default from the dict,\n            # so they get initialized with their default value\n            # except [...]\n            for param in param_defaults:\n                # [...] the ones that also have a key in the components dict.\n                # As OpenML stores different flows for ensembles with different\n                # (base-)components, in OpenML terms, these are not considered\n                # hyperparameters but rather constants (i.e., changing them would\n                # result in a different flow)\n                if param not in components.keys() and param in parameter_dict:\n                    del parameter_dict[param]\n\n        if self._is_container_module(model_class):\n            children = parameter_dict['children']\n            children = list((str(k), v) for (k, v) in children)\n            children = OrderedDict(children)\n            return model_class(children)\n\n        from .layers import Functional\n        if model_class is Functional:\n            return model_class(function=parameter_dict['function'],\n                               *parameter_dict['args'],\n                               **parameter_dict['kwargs'])\n\n        return model_class(**parameter_dict)\n\n    def _check_dependencies(self, dependencies: str) -&gt; None:\n        if not dependencies:\n            return\n\n        dependencies_list = dependencies.split('\\n')\n        for dependency_string in dependencies_list:\n            match = DEPENDENCIES_PATTERN.match(dependency_string)\n            if not match:\n                raise ValueError('Cannot parse dependency %s' % dependency_string)\n\n            dependency_name = match.group('name')\n            operation = match.group('operation')\n            version = match.group('version')\n\n            module = importlib.import_module(dependency_name)\n            required_version = LooseVersion(version)\n            installed_version = LooseVersion(module.__version__)  # type: ignore\n\n            if operation == '==':\n                check = required_version == installed_version\n            elif operation == '&gt;':\n                check = installed_version &gt; required_version\n            elif operation == '&gt;=':\n                check = (installed_version &gt; required_version\n                         or installed_version == required_version)\n            else:\n                raise NotImplementedError(\n                    'operation \\'%s\\' is not supported' % operation)\n            if not check:\n                raise ValueError('Trying to deserialize a model with dependency '\n                                 '%s not satisfied.' % dependency_string)\n\n    def _serialize_type(self, o: Any) -&gt; 'OrderedDict[str, str]':\n        mapping = {float: 'float',\n                   np.float: 'np.float',\n                   np.float32: 'np.float32',\n                   np.float64: 'np.float64',\n                   int: 'int',\n                   np.int: 'np.int',\n                   np.int32: 'np.int32',\n                   np.int64: 'np.int64'}\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret['oml-python:serialized_object'] = 'type'\n        ret['value'] = mapping[o]\n        return ret\n\n    def _deserialize_type(self, o: str) -&gt; Any:\n        mapping = {'float': float,\n                   'np.float': np.float,\n                   'np.float32': np.float32,\n                   'np.float64': np.float64,\n                   'int': int,\n                   'np.int': np.int,\n                   'np.int32': np.int32,\n                   'np.int64': np.int64}\n        return mapping[o]\n\n    def _serialize_function(self, o: Callable) -&gt; 'OrderedDict[str, str]':\n        name = o.__module__ + '.' + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret['oml-python:serialized_object'] = 'function'\n        ret['value'] = name\n        return ret\n\n    def _deserialize_function(self, name: str) -&gt; Callable:\n        module_name = name.rsplit('.', 1)\n        function_handle = getattr(importlib.import_module(module_name[0]), module_name[1])\n        return function_handle\n\n    def _serialize_methoddescriptor(self, o: Any) -&gt; 'OrderedDict[str, str]':\n        name = o.__objclass__.__module__ \\\n            + '.' + o.__objclass__.__name__ \\\n            + '.' + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret['oml-python:serialized_object'] = 'methoddescriptor'\n        ret['value'] = name\n        return ret\n\n    def _deserialize_methoddescriptor(self, name: str) -&gt; Any:\n        module_name = name.rsplit('.', 2)\n        object_handle = getattr(importlib.import_module(module_name[0]), module_name[1])\n        function_handle = getattr(object_handle, module_name[2])\n        return function_handle\n\n    def _format_external_version(\n        self,\n        model_package_name: str,\n        model_package_version_number: str,\n    ) -&gt; str:\n        return '%s==%s' % (model_package_name, model_package_version_number)\n\n    @staticmethod\n    def _get_parameter_values_recursive(param_grid: Union[Dict, List[Dict]],\n                                        parameter_name: str) -&gt; List[Any]:\n        \"\"\"\n        Returns a list of values for a given hyperparameter, encountered\n        recursively throughout the flow. (e.g., n_jobs can be defined\n        for various flows)\n\n        Parameters\n        ----------\n        param_grid: Union[Dict, List[Dict]]\n            Dict mapping from hyperparameter list to value, to a list of\n            such dicts\n\n        parameter_name: str\n            The hyperparameter that needs to be inspected\n\n        Returns\n        -------\n        List\n            A list of all values of hyperparameters with this name\n        \"\"\"\n        if isinstance(param_grid, dict):\n            result = list()\n            for param, value in param_grid.items():\n                if param.split('__')[-1] == parameter_name:\n                    result.append(value)\n            return result\n        elif isinstance(param_grid, list):\n            result = list()\n            for sub_grid in param_grid:\n                result.extend(PytorchExtension._get_parameter_values_recursive(sub_grid,\n                                                                               parameter_name))\n            return result\n        else:\n            raise ValueError('Param_grid should either be a dict or list of dicts')\n\n    ################################################################################################\n    # Methods for performing runs with extension modules\n\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is a pytorch estimator.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, torch.nn.Module)\n\n    def seed_model(self, model: Any, seed: Optional[int] = None) -&gt; Any:\n        \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n        model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Models that are already seeded will maintain the seed. In this case,\n        only integer seeds are allowed (An exception is raised when a RandomState was used as\n        seed).\n\n        Parameters\n        ----------\n        model : pytorch model\n            The model to be seeded\n        seed : int\n            The seed to initialize the RandomState with. Unseeded subcomponents\n            will be seeded with a random number from the RandomState.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        return model\n\n    def _run_model_on_fold(\n        self,\n        model: Any,\n        task: 'OpenMLTask',\n        X_train: Union[np.ndarray, scipy.sparse.spmatrix, pd.DataFrame],\n        rep_no: int,\n        fold_no: int,\n        y_train: Optional[np.ndarray] = None,\n        X_test: Optional[Union[np.ndarray, scipy.sparse.spmatrix, pd.DataFrame]] = None,\n    ) -&gt; Tuple[\n        np.ndarray,\n        np.ndarray,\n        'OrderedDict[str, float]',\n        Optional[OpenMLRunTrace],\n        Optional[Any]\n    ]:\n        \"\"\"Run a model on a repeat,fold,subsample triplet of the task and return prediction\n        information.\n\n        Furthermore, it will measure run time measures in case multi-core behaviour allows this.\n        * exact user cpu time will be measured if the number of cores is set (recursive throughout\n        the model) exactly to 1\n        * wall clock time will be measured if the number of cores is set (recursive throughout the\n        model) to any given number (but not when it is set to -1)\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        run_task_get_arff_content. Do not use this function unless you know what you are doing.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        predictions : np.ndarray\n            Model predictions.\n        probabilities :  Optional, np.ndarray\n            Predicted probabilities (only applicable for supervised classification tasks).\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : Optional, OpenMLRunTrace\n            Hyperparameter optimization trace (only applicable for supervised tasks with\n            hyperparameter optimization).\n        additional_information: Optional, Any\n            Additional information provided by the extension to be converted into additional files.\n        \"\"\"\n\n        try:\n            trainer:OpenMLTrainerModule = config.trainer\n            trainer.logger = config.logger\n        except AttributeError:\n            raise ValueError('Trainer not set to config. Please use openml_pytorch.config.trainer = trainer to set the trainer.')\n        return trainer.run_model_on_fold(model, task, X_train, rep_no, fold_no, y_train, X_test)\n\n\n    def compile_additional_information(\n            self,\n            task: 'OpenMLTask',\n            additional_information: List[Tuple[int, int, Any]]\n    ) -&gt; Dict[str, Tuple[str, str]]:\n        \"\"\"Compiles additional information provided by the extension during the runs into a final\n        set of files.\n\n        Parameters\n        ----------\n        task : OpenMLTask\n            The task the model was run on.\n        additional_information: List[Tuple[int, int, Any]]\n            A list of (fold, repetition, additional information) tuples obtained during training.\n\n        Returns\n        -------\n        files : Dict[str, Tuple[str, str]]\n            A dictionary of files with their file name and contents.\n        \"\"\"\n        return dict()\n\n    def obtain_parameter_values(\n        self,\n        flow: 'OpenMLFlow',\n        model: Any = None,\n    ) -&gt; List[Dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n        openml.flows.functions._check_flow_for_server_id(flow)\n\n        def get_flow_dict(_flow):\n            flow_map = {_flow.name: _flow.flow_id}\n            for subflow in _flow.components:\n                flow_map.update(get_flow_dict(_flow.components[subflow]))\n            return flow_map\n\n        def extract_parameters(_flow, _flow_dict, component_model,\n                               _main_call=False, main_id=None):\n            def is_subcomponent_specification(values):\n                # checks whether the current value can be a specification of\n                # subcomponents, as for example the value for steps parameter\n                # (in Pipeline) or transformers parameter (in\n                # ColumnTransformer). These are always lists/tuples of lists/\n                # tuples, size bigger than 2 and an OpenMLFlow item involved.\n                if not isinstance(values, (tuple, list)):\n                    return False\n                for item in values:\n                    if not isinstance(item, (tuple, list)):\n                        return False\n                    if len(item) &lt; 2:\n                        return False\n                    if not isinstance(item[1], openml.flows.OpenMLFlow):\n                        return False\n                return True\n\n            # _flow is openml flow object, _param dict maps from flow name to flow\n            # id for the main call, the param dict can be overridden (useful for\n            # unit tests / sentinels) this way, for flows without subflows we do\n            # not have to rely on _flow_dict\n            exp_parameters = set(_flow.parameters)\n            exp_components = set(_flow.components)\n            model_parameters = set([mp for mp in self._get_module_descriptors(component_model)\n                                    if '__' not in mp])\n            if len((exp_parameters | exp_components) ^ model_parameters) != 0:\n                flow_params = sorted(exp_parameters | exp_components)\n                model_params = sorted(model_parameters)\n                raise ValueError('Parameters of the model do not match the '\n                                 'parameters expected by the '\n                                 'flow:\\nexpected flow parameters: '\n                                 '%s\\nmodel parameters: %s' % (flow_params,\n                                                               model_params))\n\n            _params = []\n            for _param_name in _flow.parameters:\n                _current = OrderedDict()\n                _current['oml:name'] = _param_name\n\n                current_param_values = self.model_to_flow(\n                    self._get_module_descriptors(component_model)[_param_name])\n\n                # Try to filter out components (a.k.a. subflows) which are\n                # handled further down in the code (by recursively calling\n                # this function)!\n                if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                    continue\n\n                if is_subcomponent_specification(current_param_values):\n                    # complex parameter value, with subcomponents\n                    parsed_values = list()\n                    for subcomponent in current_param_values:\n                        if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                            raise ValueError('Component reference should be '\n                                             'size {2,3}. ')\n\n                        subcomponent_identifier = subcomponent[0]\n                        subcomponent_flow = subcomponent[1]\n                        if not isinstance(subcomponent_identifier, str):\n                            raise TypeError('Subcomponent identifier should be '\n                                            'string')\n                        if not isinstance(subcomponent_flow,\n                                          openml.flows.OpenMLFlow):\n                            raise TypeError('Subcomponent flow should be string')\n\n                        current = {\n                            \"oml-python:serialized_object\": \"component_reference\",\n                            \"value\": {\n                                \"key\": subcomponent_identifier,\n                                \"step_name\": subcomponent_identifier\n                            }\n                        }\n                        if len(subcomponent) == 3:\n                            if not isinstance(subcomponent[2], list):\n                                raise TypeError('Subcomponent argument should be'\n                                                'list')\n                            current['value']['argument_1'] = subcomponent[2]\n                        parsed_values.append(current)\n                    parsed_values = json.dumps(parsed_values)\n                else:\n                    # vanilla parameter value\n                    parsed_values = json.dumps(current_param_values)\n\n                _current['oml:value'] = parsed_values\n                if _main_call:\n                    _current['oml:component'] = main_id\n                else:\n                    _current['oml:component'] = _flow_dict[_flow.name]\n                _params.append(_current)\n\n            for _identifier in _flow.components:\n                subcomponent_model = self._get_module_descriptors(component_model)[_identifier]\n                _params.extend(extract_parameters(_flow.components[_identifier],\n                                                  _flow_dict, subcomponent_model))\n            return _params\n\n        flow_dict = get_flow_dict(flow)\n        model = model if model is not None else flow.model\n        parameters = extract_parameters(flow, flow_dict, model, True, flow.flow_id)\n\n        return parameters\n\n    def _openml_param_name_to_pytorch(\n        self,\n        openml_parameter: openml.setups.OpenMLParameter,\n        flow: OpenMLFlow,\n    ) -&gt; str:\n        \"\"\"\n        Converts the name of an OpenMLParameter into the pytorch name, given a flow.\n\n        Parameters\n        ----------\n        openml_parameter: OpenMLParameter\n            The parameter under consideration\n\n        flow: OpenMLFlow\n            The flow that provides context.\n\n        Returns\n        -------\n        pytorch_parameter_name: str\n            The name the parameter will have once used in pytorch\n        \"\"\"\n        if not isinstance(openml_parameter, openml.setups.OpenMLParameter):\n            raise ValueError('openml_parameter should be an instance of OpenMLParameter')\n        if not isinstance(flow, OpenMLFlow):\n            raise ValueError('flow should be an instance of OpenMLFlow')\n\n        flow_structure = flow.get_structure('name')\n        if openml_parameter.flow_name not in flow_structure:\n            raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not correspond. ')\n        name = openml_parameter.flow_name  # for PEP8\n        return '__'.join(flow_structure[name] + [openml_parameter.parameter_name])\n\n    ################################################################################################\n    # Methods for hyperparameter optimization\n\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n        optimization model (UNUSED)\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        return model\n\n\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained\n        Parameters\n        ----------\n        model : Any\n        Returns\n        -------\n        bool\n        \"\"\"\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>classmethod</code>","text":"<p>Check whether a given describes a Pytorch estimator.</p> <p>This is done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>@classmethod\ndef can_handle_flow(cls, flow: 'OpenMLFlow') -&gt; bool:\n    \"\"\"Check whether a given describes a Pytorch estimator.\n\n    This is done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return cls._is_pytorch_flow(flow)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>classmethod</code>","text":"<p>Check whether a model is an instance of <code>torch.nn.Module</code>.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>@classmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model is an instance of ``torch.nn.Module``.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    from torch.nn import Module\n    return isinstance(model, Module)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained\n    Parameters\n    ----------\n    model : Any\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.compile_additional_information","title":"<code>compile_additional_information(task, additional_information)</code>","text":"<p>Compiles additional information provided by the extension during the runs into a final set of files.</p> <p>Parameters:</p> Name Type Description Default <code>task</code> <code>OpenMLTask</code> <p>The task the model was run on.</p> required <code>additional_information</code> <code>List[Tuple[int, int, Any]]</code> <p>A list of (fold, repetition, additional information) tuples obtained during training.</p> required <p>Returns:</p> Name Type Description <code>files</code> <code>Dict[str, Tuple[str, str]]</code> <p>A dictionary of files with their file name and contents.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def compile_additional_information(\n        self,\n        task: 'OpenMLTask',\n        additional_information: List[Tuple[int, int, Any]]\n) -&gt; Dict[str, Tuple[str, str]]:\n    \"\"\"Compiles additional information provided by the extension during the runs into a final\n    set of files.\n\n    Parameters\n    ----------\n    task : OpenMLTask\n        The task the model was run on.\n    additional_information: List[Tuple[int, int, Any]]\n        A list of (fold, repetition, additional information) tuples obtained during training.\n\n    Returns\n    -------\n    files : Dict[str, Tuple[str, str]]\n        A dictionary of files with their file name and contents.\n    \"\"\"\n    return dict()\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.create_setup_string","title":"<code>create_setup_string(model)</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def create_setup_string(self, model: Any) -&gt; str:\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n    run_environment = \" \".join(self.get_version_information())\n    return run_environment + \" \" + str(model)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False)</code>","text":"<p>Initializes a Pytorch model based on a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>mixed</code> <p>the object to deserialize (can be flow object, or any serialized parameter value that is accepted by)</p> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <p>Returns:</p> Type Description <code>mixed</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def flow_to_model(self, flow: 'OpenMLFlow', initialize_with_defaults: bool = False) -&gt; Any:\n    \"\"\"Initializes a Pytorch model based on a flow.\n\n    Parameters\n    ----------\n    flow : mixed\n        the object to deserialize (can be flow object, or any serialized\n        parameter value that is accepted by)\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    Returns\n    -------\n    mixed\n    \"\"\"\n    return self._deserialize_pytorch(flow, initialize_with_defaults=initialize_with_defaults)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.get_version_information","title":"<code>get_version_information()</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Libraries listed are <code>Python</code>, <code>pytorch</code>, <code>numpy</code> and <code>scipy</code>.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def get_version_information(self) -&gt; List[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Libraries listed are ``Python``, ``pytorch``, ``numpy`` and ``scipy``.\n\n    Returns\n    -------\n    List\n    \"\"\"\n\n    # This can possibly be done by a package such as pyxb, but I could not get\n    # it to work properly.\n    import scipy\n    import numpy\n\n    major, minor, micro, _, _ = sys.version_info\n    python_version = 'Python_{}.'.format(\n        \".\".join([str(major), str(minor), str(micro)]))\n    pytorch_version = 'Torch_{}.'.format(torch.__version__)\n    numpy_version = 'NumPy_{}.'.format(numpy.__version__)\n    scipy_version = 'SciPy_{}.'.format(scipy.__version__)\n    pytorch_version_formatted = pytorch_version.replace('+','_')\n    return [python_version, pytorch_version_formatted, numpy_version, scipy_version]\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>","text":"<p>Instantiate a <code>base_estimator</code> which can be searched over by the hyperparameter optimization model (UNUSED)</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n    optimization model (UNUSED)\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    return model\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.is_estimator","title":"<code>is_estimator(model)</code>","text":"<p>Check whether the given model is a pytorch estimator.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is a pytorch estimator.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return isinstance(model, torch.nn.Module)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.model_to_flow","title":"<code>model_to_flow(model, custom_name=None)</code>","text":"<p>Transform a Pytorch model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def model_to_flow(self, model: Any, custom_name: Optional[str] = None) -&gt; 'OpenMLFlow':\n    \"\"\"Transform a Pytorch model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n    # Necessary to make pypy not complain about all the different possible return types\n    return self._serialize_pytorch(model, custom_name)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def obtain_parameter_values(\n    self,\n    flow: 'OpenMLFlow',\n    model: Any = None,\n) -&gt; List[Dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n    openml.flows.functions._check_flow_for_server_id(flow)\n\n    def get_flow_dict(_flow):\n        flow_map = {_flow.name: _flow.flow_id}\n        for subflow in _flow.components:\n            flow_map.update(get_flow_dict(_flow.components[subflow]))\n        return flow_map\n\n    def extract_parameters(_flow, _flow_dict, component_model,\n                           _main_call=False, main_id=None):\n        def is_subcomponent_specification(values):\n            # checks whether the current value can be a specification of\n            # subcomponents, as for example the value for steps parameter\n            # (in Pipeline) or transformers parameter (in\n            # ColumnTransformer). These are always lists/tuples of lists/\n            # tuples, size bigger than 2 and an OpenMLFlow item involved.\n            if not isinstance(values, (tuple, list)):\n                return False\n            for item in values:\n                if not isinstance(item, (tuple, list)):\n                    return False\n                if len(item) &lt; 2:\n                    return False\n                if not isinstance(item[1], openml.flows.OpenMLFlow):\n                    return False\n            return True\n\n        # _flow is openml flow object, _param dict maps from flow name to flow\n        # id for the main call, the param dict can be overridden (useful for\n        # unit tests / sentinels) this way, for flows without subflows we do\n        # not have to rely on _flow_dict\n        exp_parameters = set(_flow.parameters)\n        exp_components = set(_flow.components)\n        model_parameters = set([mp for mp in self._get_module_descriptors(component_model)\n                                if '__' not in mp])\n        if len((exp_parameters | exp_components) ^ model_parameters) != 0:\n            flow_params = sorted(exp_parameters | exp_components)\n            model_params = sorted(model_parameters)\n            raise ValueError('Parameters of the model do not match the '\n                             'parameters expected by the '\n                             'flow:\\nexpected flow parameters: '\n                             '%s\\nmodel parameters: %s' % (flow_params,\n                                                           model_params))\n\n        _params = []\n        for _param_name in _flow.parameters:\n            _current = OrderedDict()\n            _current['oml:name'] = _param_name\n\n            current_param_values = self.model_to_flow(\n                self._get_module_descriptors(component_model)[_param_name])\n\n            # Try to filter out components (a.k.a. subflows) which are\n            # handled further down in the code (by recursively calling\n            # this function)!\n            if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                continue\n\n            if is_subcomponent_specification(current_param_values):\n                # complex parameter value, with subcomponents\n                parsed_values = list()\n                for subcomponent in current_param_values:\n                    if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                        raise ValueError('Component reference should be '\n                                         'size {2,3}. ')\n\n                    subcomponent_identifier = subcomponent[0]\n                    subcomponent_flow = subcomponent[1]\n                    if not isinstance(subcomponent_identifier, str):\n                        raise TypeError('Subcomponent identifier should be '\n                                        'string')\n                    if not isinstance(subcomponent_flow,\n                                      openml.flows.OpenMLFlow):\n                        raise TypeError('Subcomponent flow should be string')\n\n                    current = {\n                        \"oml-python:serialized_object\": \"component_reference\",\n                        \"value\": {\n                            \"key\": subcomponent_identifier,\n                            \"step_name\": subcomponent_identifier\n                        }\n                    }\n                    if len(subcomponent) == 3:\n                        if not isinstance(subcomponent[2], list):\n                            raise TypeError('Subcomponent argument should be'\n                                            'list')\n                        current['value']['argument_1'] = subcomponent[2]\n                    parsed_values.append(current)\n                parsed_values = json.dumps(parsed_values)\n            else:\n                # vanilla parameter value\n                parsed_values = json.dumps(current_param_values)\n\n            _current['oml:value'] = parsed_values\n            if _main_call:\n                _current['oml:component'] = main_id\n            else:\n                _current['oml:component'] = _flow_dict[_flow.name]\n            _params.append(_current)\n\n        for _identifier in _flow.components:\n            subcomponent_model = self._get_module_descriptors(component_model)[_identifier]\n            _params.extend(extract_parameters(_flow.components[_identifier],\n                                              _flow_dict, subcomponent_model))\n        return _params\n\n    flow_dict = get_flow_dict(flow)\n    model = model if model is not None else flow.model\n    parameters = extract_parameters(flow, flow_dict, model, True, flow.flow_id)\n\n    return parameters\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.seed_model","title":"<code>seed_model(model, seed=None)</code>","text":"<p>Set the random state of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is raised when a RandomState was used as seed).</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>pytorch model</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> <p>The seed to initialize the RandomState with. Unseeded subcomponents will be seeded with a random number from the RandomState.</p> <code>None</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def seed_model(self, model: Any, seed: Optional[int] = None) -&gt; Any:\n    \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n    model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Models that are already seeded will maintain the seed. In this case,\n    only integer seeds are allowed (An exception is raised when a RandomState was used as\n    seed).\n\n    Parameters\n    ----------\n    model : pytorch model\n        The model to be seeded\n    seed : int\n        The seed to initialize the RandomState with. Unseeded subcomponents\n        will be seeded with a random number from the RandomState.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    return model\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/","title":"Trainer","text":"<p>This module provides classes and methods to facilitate the configuration, data handling, training, and evaluation of machine learning models using PyTorch and OpenML datasets. The functionalities include: - Generation of default configurations for models. - Handling of image and tabular data. - Training and evaluating machine learning models. - Exporting trained models to ONNX format. - Managing data transformations and loaders.</p> <p>This module provides classes and methods to facilitate the configuration, data handling, training, and evaluation of machine learning models using PyTorch and OpenML datasets. The functionalities include: - Generation of default configurations for models. - Handling of image and tabular data. - Training and evaluating machine learning models. - Exporting trained models to ONNX format. - Managing data transformations and loaders.</p>"},{"location":"pytorch/API%20reference/Trainer/#trainer.BaseDataHandler","title":"<code>BaseDataHandler</code>","text":"<p>BaseDataHandler class is an abstract base class for data handling operations.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class BaseDataHandler:\n    \"\"\"\n        BaseDataHandler class is an abstract base class for data handling operations.\n    \"\"\"\n    def prepare_data(self, X_train, y_train, X_val, y_val, data_config=None):\n        raise NotImplementedError\n\n    def prepare_test_data(self, X_test, data_config=None):\n        raise NotImplementedError\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DataContainer","title":"<code>DataContainer</code>","text":"<p>class DataContainer:     A class to contain the training, validation, and test data loaders. This just makes it easier to access them when required.</p> <pre><code>Attributes:\ntrain_dl: DataLoader object for the training data.\nvalid_dl: DataLoader object for the validation data.\ntest_dl: Optional DataLoader object for the test data.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class DataContainer:\n    \"\"\"\n    class DataContainer:\n        A class to contain the training, validation, and test data loaders. This just makes it easier to access them when required.\n\n        Attributes:\n        train_dl: DataLoader object for the training data.\n        valid_dl: DataLoader object for the validation data.\n        test_dl: Optional DataLoader object for the test data.\n    \"\"\"\n    def __init__(self, train_dl, valid_dl, test_dl=None):\n        self.train_dl, self.valid_dl = train_dl, valid_dl\n        self.test_dl = test_dl\n\n    @property\n    def train_ds(self):\n        return self.train_dl.dataset\n\n    @property\n    def valid_ds(self):\n        return self.valid_dl.dataset\n\n    @property\n    def test_ds(self):\n        return self.test_dl.dataset\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator","title":"<code>DefaultConfigGenerator</code>","text":"<p>DefaultConfigGenerator class provides various methods to generate default configurations.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class DefaultConfigGenerator:\n    \"\"\"\n    DefaultConfigGenerator class provides various methods to generate default configurations.\n    \"\"\"\n\n    @staticmethod\n    def _default_criterion_gen(task: OpenMLTask) -&gt; torch.nn.Module:\n        \"\"\"\n        _default_criterion_gen returns a criterion based on the task type - regressions use\n        torch.nn.SmoothL1Loss while classifications use torch.nn.CrossEntropyLoss\n        \"\"\"\n        if isinstance(task, OpenMLRegressionTask):\n            return torch.nn.SmoothL1Loss()\n        elif isinstance(task, OpenMLClassificationTask):\n            return torch.nn.CrossEntropyLoss()\n        else:\n            raise ValueError(task)\n\n    @staticmethod\n    def _default_optimizer_gen(model: torch.nn.Module, _: OpenMLTask):\n        \"\"\"\n        _default_optimizer_gen returns the torch.optim.Adam optimizer for the given model\n        \"\"\"\n        return torch.optim.Adam\n\n    @staticmethod\n    def _default_scheduler_gen(optim, _: OpenMLTask) -&gt; Any:\n        \"\"\"\n        _default_scheduler_gen returns the torch.optim.lr_scheduler.ReduceLROnPlateau scheduler for the given optimizer\n        \"\"\"\n        return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optim)\n\n    @staticmethod\n    def _default_predict(output: torch.Tensor, task: OpenMLTask) -&gt; torch.Tensor:\n        \"\"\"\n        _default_predict turns the outputs into predictions by returning the argmax of the output tensor for classification tasks, and by flattening the prediction in case of the regression\n        \"\"\"\n        output_axis = output.dim() - 1\n        if isinstance(task, OpenMLClassificationTask):\n            output = torch.argmax(output, dim=output_axis)\n        elif isinstance(task, OpenMLRegressionTask):\n            output = output.view(-1)\n        else:\n            raise ValueError(task)\n        return output\n\n    @staticmethod\n    def _default_predict_proba(\n        output: torch.Tensor, task: OpenMLTask\n    ) -&gt; torch.Tensor:\n        \"\"\"\n        _default_predict_proba turns the outputs into probabilities using softmax\n        \"\"\"\n        output_axis = output.dim() - 1\n        output = output.softmax(dim=output_axis)\n        return output\n\n    @staticmethod\n    def _default_sanitize(tensor: torch.Tensor) -&gt; torch.Tensor:\n        \"\"\"\n        _default sanitizer replaces NaNs with 1e-6\n        \"\"\"\n        tensor = torch.where(\n            torch.isnan(tensor), torch.ones_like(tensor) * torch.tensor(1e-6), tensor\n        )\n        return tensor\n\n    @staticmethod\n    def _default_retype_labels(\n         tensor: torch.Tensor, task: OpenMLTask\n    ) -&gt; torch.Tensor:\n        \"\"\"\n        _default_retype_labels changes the type of the tensor to long for classification tasks and to float for regression tasks\n        \"\"\"\n        if isinstance(task, OpenMLClassificationTask):\n            return tensor.long()\n        elif isinstance(task, OpenMLRegressionTask):\n            return tensor.float()\n        else:\n            raise ValueError(task)\n\n    def get_device(\n        self,\n    ):\n        \"\"\"\n        Checks if a GPU is available and returns the device to be used for training (cuda, mps or cpu)\n        \"\"\"\n        if torch.cuda.is_available():\n            device = torch.device(\"cuda\")\n        elif torch.backends.mps.is_available() and torch.backends.mps.is_built():\n            device = torch.device(\"mps\")\n        else:\n            device = torch.device(\"cpu\")\n\n        return device\n\n    def default_image_transform(self):\n        return Compose(\n            [\n                ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n                Lambda(convert_to_rgb),  # Convert PIL Image to RGB if it's not already.\n                Resize((128, 128)),  # Resize the image.\n                ToTensor(),  # Convert the PIL Image back to a tensor.\n            ]\n        )\n\n    def return_model_config(self):\n        \"\"\"\n        Returns a configuration object for the model\n        \"\"\"\n\n        return SimpleNamespace(\n            device=self.get_device(),\n            criterion=self._default_criterion_gen,\n            optimizer_gen=self._default_optimizer_gen,\n            scheduler_gen=self._default_scheduler_gen,\n            # predict turns the outputs of the model into actual predictions\n            predict=self._default_predict,  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n            # predict_proba turns the outputs of the model into probabilities for each class\n            predict_proba=self._default_predict_proba,  # type: Callable[[torch.Tensor], torch.Tensor]\n            # epoch_count represents the number of epochs the model should be trained for\n            epoch_count=3,  # type: int,\n            # progress_callback=(\n            #     self._default_progress_callback\n            # ),  # type: Callable[[int, int, int, int, float, float], None]\n            # enable progress bar\n            verbose=True,\n        )\n\n    def return_data_config(self):\n        \"\"\"\n        Returns a configuration object for the data\n        \"\"\"\n        return SimpleNamespace(\n            type_of_data=\"image\",\n            perform_validation=False,\n            # progress_callback is called when a training step is finished, in order to report the current progress\n            # sanitize sanitizes the input data in order to ensure that models can be trained safely\n            sanitize=self._default_sanitize,  # type: Callable[[torch.Tensor], torch.Tensor]\n            # retype_labels changes the types of the labels in order to ensure type compatibility\n            retype_labels=(\n                self._default_retype_labels\n            ),  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n            # image_size is the size of the images that are fed into the model\n            image_size=128,\n            # batch_size represents the processing batch size for training\n            batch_size=64,  # type: int\n            data_augmentation=None,\n            validation_split=0.1,\n            transform=self.default_image_transform(),\n        )\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator.get_device","title":"<code>get_device()</code>","text":"<p>Checks if a GPU is available and returns the device to be used for training (cuda, mps or cpu)</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def get_device(\n    self,\n):\n    \"\"\"\n    Checks if a GPU is available and returns the device to be used for training (cuda, mps or cpu)\n    \"\"\"\n    if torch.cuda.is_available():\n        device = torch.device(\"cuda\")\n    elif torch.backends.mps.is_available() and torch.backends.mps.is_built():\n        device = torch.device(\"mps\")\n    else:\n        device = torch.device(\"cpu\")\n\n    return device\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator.return_data_config","title":"<code>return_data_config()</code>","text":"<p>Returns a configuration object for the data</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def return_data_config(self):\n    \"\"\"\n    Returns a configuration object for the data\n    \"\"\"\n    return SimpleNamespace(\n        type_of_data=\"image\",\n        perform_validation=False,\n        # progress_callback is called when a training step is finished, in order to report the current progress\n        # sanitize sanitizes the input data in order to ensure that models can be trained safely\n        sanitize=self._default_sanitize,  # type: Callable[[torch.Tensor], torch.Tensor]\n        # retype_labels changes the types of the labels in order to ensure type compatibility\n        retype_labels=(\n            self._default_retype_labels\n        ),  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n        # image_size is the size of the images that are fed into the model\n        image_size=128,\n        # batch_size represents the processing batch size for training\n        batch_size=64,  # type: int\n        data_augmentation=None,\n        validation_split=0.1,\n        transform=self.default_image_transform(),\n    )\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator.return_model_config","title":"<code>return_model_config()</code>","text":"<p>Returns a configuration object for the model</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def return_model_config(self):\n    \"\"\"\n    Returns a configuration object for the model\n    \"\"\"\n\n    return SimpleNamespace(\n        device=self.get_device(),\n        criterion=self._default_criterion_gen,\n        optimizer_gen=self._default_optimizer_gen,\n        scheduler_gen=self._default_scheduler_gen,\n        # predict turns the outputs of the model into actual predictions\n        predict=self._default_predict,  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n        # predict_proba turns the outputs of the model into probabilities for each class\n        predict_proba=self._default_predict_proba,  # type: Callable[[torch.Tensor], torch.Tensor]\n        # epoch_count represents the number of epochs the model should be trained for\n        epoch_count=3,  # type: int,\n        # progress_callback=(\n        #     self._default_progress_callback\n        # ),  # type: Callable[[int, int, int, int, float, float], None]\n        # enable progress bar\n        verbose=True,\n    )\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.OpenMLImageHandler","title":"<code>OpenMLImageHandler</code>","text":"<p>               Bases: <code>BaseDataHandler</code></p> <p>OpenMLImageHandler is a class that extends BaseDataHandler to handle image data from OpenML datasets.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class OpenMLImageHandler(BaseDataHandler):\n    \"\"\"\n        OpenMLImageHandler is a class that extends BaseDataHandler to handle image data from OpenML datasets.\n    \"\"\"\n    def prepare_data(self, X_train, y_train, X_val, y_val, data_config = None):\n        train = OpenMLImageDataset(\n            image_dir=data_config.file_dir,\n            X=X_train,\n            y=y_train,\n            transform_x=data_config.transform,\n            image_size=data_config.image_size,\n        )\n        val = OpenMLImageDataset(\n            image_dir=data_config.file_dir,\n            X=X_val,\n            y=y_val,\n            transform_x=data_config.transform,\n            image_size=data_config.image_size,\n        )\n        return train, val\n\n    def prepare_test_data(self, X_test, data_config = None):\n        test = OpenMLImageDataset(\n            image_dir=data_config.file_dir,\n            X=X_test,\n            y=None,\n            transform_x=data_config.transform,\n            image_size=data_config.image_size,\n        )\n        return test\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.OpenMLTabularHandler","title":"<code>OpenMLTabularHandler</code>","text":"<p>               Bases: <code>BaseDataHandler</code></p> <p>OpenMLTabularHandler is a class that extends BaseDataHandler to handle tabular data from OpenML datasets.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class OpenMLTabularHandler(BaseDataHandler):\n    \"\"\"\n    OpenMLTabularHandler is a class that extends BaseDataHandler to handle tabular data from OpenML datasets.\n    \"\"\"\n    def prepare_data(self, X_train, y_train, X_val, y_val, data_config = None):\n        train = OpenMLTabularDataset(X=X_train, y=y_train)\n        val = OpenMLTabularDataset(X=X_val, y=y_val)\n        return train, val\n\n    def prepare_test_data(self, X_test, data_config = None):\n        test = OpenMLTabularDataset(X=X_test, y=None)\n        return test\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.OpenMLTrainerModule","title":"<code>OpenMLTrainerModule</code>","text":"Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class OpenMLTrainerModule:\n\n    def _default_progress_callback(\n        self, fold: int, rep: int, epoch: int, step: int, loss: float, accuracy: float\n    ):\n        # todo : move this into callback\n        \"\"\"\n                _default_progress_callback reports the current fold, rep, epoch, step and loss for every\n        training iteration to the default logger\n        \"\"\"\n        self.logger.info(\n            \"[%d, %d, %d, %d] loss: %.4f, accuracy: %.4f\"\n            % (fold, rep, epoch, step, loss, accuracy)\n        )\n    def __init__(\n        self,\n        data_module: OpenMLDataModule,\n        callbacks: List[Callback] = [],\n        **kwargs,\n    ):\n        self.config_gen = DefaultConfigGenerator()\n        self.model_config = self.config_gen.return_model_config()\n        self.data_module = data_module\n        self.callbacks = callbacks\n\n        self.config = SimpleNamespace(\n            **{**self.model_config.__dict__, **self.data_module.data_config.__dict__}\n        )\n        # update the config with the user defined values\n        self.config.__dict__.update(kwargs)\n        self.config.progress_callback = self._default_progress_callback\n        self.logger: logging.Logger = logging.getLogger(__name__)\n\n        self.user_defined_measures = OrderedDict()\n        # self.callbacks.append(LoggingCallback(self.logger, print_output=False))\n        self.loss = 0\n        self.training_state = True\n\n        self.phases = [0.2, 0.8]\n        self.scheds = combine_scheds(\n            self.phases, [sched_cos(1e-4, 5e-3), sched_cos(5e-3, 1e-3)]\n        )\n\n        self.cbfs = [\n            Recorder,\n            partial(AvgStatsCallBack, [accuracy]),\n            partial(ParamScheduler, \"lr\", self.scheds),\n            # TensorBoardCallback(),\n        ]\n\n    def _onnx_export(self, model_copy):\n        f = io.BytesIO()\n        torch.onnx.export(model_copy, sample_input, f)\n        onnx_model = onnx.load_model_from_string(f.getvalue())\n        onnx_ = onnx_model.SerializeToString()\n        return onnx_\n\n    def run_model_on_fold(\n        self,\n        model: torch.nn.Module,\n        task: OpenMLTask,\n        X_train: pd.DataFrame,\n        rep_no: int,\n        fold_no: int,\n        y_train: Optional[pd.Series],\n        X_test: pd.DataFrame,\n    ) -&gt; Tuple[np.ndarray, Optional[np.ndarray], OrderedDict, Optional[Any]]:\n\n        # if task has no class labels, we assign the class labels to be the unique values in the training set\n        if task.class_labels is None:\n            task.class_labels = y_train.unique()\n\n        self.add_callbacks()\n\n        self.model = copy.deepcopy(model)\n\n        try:\n            data, model_classes = self.run_training(task, X_train, y_train, X_test)\n\n        except AttributeError as e:\n            # typically happens when training a regressor8 on classification task\n            raise PyOpenMLError(str(e))\n\n        # In supervised learning this returns the predictions for Y\n        pred_y, proba_y = self.run_evaluation(task, data, model_classes)\n\n        # Convert predictions to class labels\n        if task.class_labels is not None:\n            pred_y = [task.class_labels[i] for i in pred_y]\n\n        # Convert model to onnx\n        onnx_ = self._onnx_export(self.model)\n\n        global last_models\n        last_models = onnx_\n\n        return pred_y, proba_y, self.user_defined_measures, None\n\n    def check_config(self):\n        raise NotImplementedError\n\n    def _prediction_to_probabilities(\n        self, y: np.ndarray, classes: List[Any]\n    ) -&gt; np.ndarray:\n        \"\"\"Transforms predicted probabilities to match with OpenML class indices.\n\n        Parameters\n        ----------\n        y : np.ndarray\n            Predicted probabilities (possibly omitting classes if they were not present in the\n            training data).\n        model_classes : list\n            List of classes known_predicted by the model, ordered by their index.\n\n        Returns\n        -------\n        np.ndarray\n        \"\"\"\n        # y: list or numpy array of predictions\n        # model_classes: mapping from original array id to\n        # prediction index id\n        if not isinstance(classes, list):\n            raise ValueError(\n                \"please convert model classes to list prior to \" \"calling this fn\"\n            )\n        result = np.zeros((len(y), len(classes)), dtype=np.float32)\n        for obs, prediction_idx in enumerate(y):\n            result[obs][prediction_idx] = 1.0\n        return result\n\n    def run_evaluation(self, task, data, model_classes):\n        if isinstance(task, OpenMLSupervisedTask):\n            self.model.eval()\n            pred_y = self.pred_test(task, self.model, data.test_dl, self.config.predict)\n        else:\n            raise ValueError(task)\n\n        if isinstance(task, OpenMLClassificationTask):\n            try:\n                self.model.eval()\n                proba_y = self.pred_test(\n                    task, self.model, data.test_dl, self.config.predict_proba\n                )\n\n            except AttributeError:\n                if task.class_labels is not None:\n                    proba_y = self._prediction_to_probabilities(\n                        pred_y, list(task.class_labels)\n                    )\n                else:\n                    raise ValueError(\"The task has no class labels\")\n\n            if task.class_labels is not None:\n                if proba_y.shape[1] != len(task.class_labels):\n                    # Remap the probabilities in case there was a class missing\n                    # at training time. By default, the classification targets\n                    # are mapped to be zero-based indices to the actual classes.\n                    # Therefore, the model_classes contain the correct indices to\n                    # the correct probability array. Example:\n                    # classes in the dataset: 0, 1, 2, 3, 4, 5\n                    # classes in the training set: 0, 1, 2, 4, 5\n                    # then we need to add a column full of zeros into the probabilities\n                    # for class 3 because the rest of the library expects that the\n                    # probabilities are ordered the same way as the classes are ordered).\n                    proba_y_new = np.zeros((proba_y.shape[0], len(task.class_labels)))\n                    for idx, model_class in enumerate(model_classes):\n                        proba_y_new[:, model_class] = proba_y[:, idx]\n                    proba_y = proba_y_new\n\n                if proba_y.shape[1] != len(task.class_labels):\n                    message = \"Estimator only predicted for {}/{} classes!\".format(\n                        proba_y.shape[1],\n                        len(task.class_labels),\n                    )\n                    warnings.warn(message)\n                    self.logger.warning(message)\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            proba_y = None\n\n        else:\n            raise TypeError(type(task))\n        return pred_y, proba_y\n\n    def run_training(self, task, X_train, y_train, X_test):\n        if isinstance(task, OpenMLSupervisedTask) or isinstance(\n            task, OpenMLClassificationTask\n        ):\n            self.opt = self.config.optimizer_gen(self.model, task)(\n                self.model.parameters()\n            )\n\n            self.criterion = self.config.criterion(task)\n            self.device = self.config.device\n\n            if self.config.device != \"cpu\":\n                self.criterion = self.criterion.to(self.config.device)\n\n            data, model_classes = self.data_module.get_data(\n                X_train, y_train, X_test, task\n            )\n            self.learn = Learner(\n                self.model,\n                self.opt,\n                self.criterion,\n                data,\n                model_classes,\n            )\n            self.learn.device = self.device\n            self.learn.model.to(self.device)\n            gc.collect()\n\n            self.runner = ModelRunner(cb_funcs=self.cbfs)\n            self.learn.model.train()\n            self.runner.fit(epochs=self.config.epoch_count, learn=self.learn)\n            self.learn.model.eval()\n\n            print(\"Loss\", self.runner.loss)\n        return data, model_classes\n\n    def add_callbacks(self):\n        if self.callbacks is not None and len(self.callbacks) &gt; 0:\n            for callback in self.callbacks:\n                if callback not in self.cbfs:\n                    self.cbfs.append(callback)\n\n    def pred_test(self, task, model_copy, test_loader, predict_func):\n        probabilities = []\n        for batch_idx, inputs in enumerate(test_loader):\n            inputs = self.config.sanitize(inputs)\n            # if torch.cuda.is_available():\n            inputs = inputs.to(self.config.device)\n\n            # Perform inference on the batch\n            pred_y_batch = model_copy(inputs)\n            pred_y_batch = predict_func(pred_y_batch, task)\n            pred_y_batch = pred_y_batch.cpu().detach().numpy()\n\n            probabilities.append(pred_y_batch)\n\n            # Concatenate probabilities from all batches\n        pred_y = np.concatenate(probabilities, axis=0)\n        return pred_y\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.convert_to_rgb","title":"<code>convert_to_rgb(image)</code>","text":"<p>Converts an image to RGB mode if it is not already in that mode.</p> <p>Parameters: image (PIL.Image): The image to be converted.</p> <p>Returns: PIL.Image: The converted image in RGB mode.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def convert_to_rgb(image):\n    \"\"\"\n        Converts an image to RGB mode if it is not already in that mode.\n\n        Parameters:\n        image (PIL.Image): The image to be converted.\n\n        Returns:\n        PIL.Image: The converted image in RGB mode.\n    \"\"\"\n    if image.mode != \"RGB\":\n        return image.convert(\"RGB\")\n    return image\n</code></pre>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/","title":"Create dataset and task - tiniest imagenet","text":"In\u00a0[2]: Copied! <pre>import openml\n\nimport numpy as np\nimport pandas as pd\nimport sklearn.datasets\n\nimport openml\nfrom openml.datasets.functions import create_dataset\nimport os\nimport requests\nimport zipfile\nimport glob\n</pre> import openml  import numpy as np import pandas as pd import sklearn.datasets  import openml from openml.datasets.functions import create_dataset import os import requests import zipfile import glob In\u00a0[9]: Copied! <pre>def create_tiny_imagenet():\n    dir_name = \"datasets\"\n    os.makedirs(dir_name, exist_ok=True)\n\n    # download the dataset\n    url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"\n    r = requests.get(url, stream=True)\n\n    if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):\n        with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:\n            f.write(r.content)\n\n        with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:\n            zip_ref.extractall(f\"{dir_name}/\")\n    ## recusively find all the images\n    image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")\n    ## remove the first part of the path\n    image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]\n    ## create a dataframe with the image path and the label\n    label_func = lambda x: x.split(\"/\")[2]\n    df = pd.DataFrame(image_paths, columns=[\"image_path\"])\n    df[\"label\"] = df[\"image_path\"].apply(label_func)\n    ## encode the labels as integers\n    # df[\"Class_encoded\"] = pd.factorize(df[\"label\"])[0]\n\n    ## encode types\n    df[\"image_path\"] = df[\"image_path\"].astype(\"string\")\n    df[\"label\"] = df[\"label\"].astype(\"string\")\n\n\n    name = \"tiny-imagenet-200\"\n    attribute_names = df.columns\n    description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"\n    paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"\n    citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")\n\n    tinyim = create_dataset(\n        name = name,\n        description = description,\n        creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        collection_date = \"2017\",\n        language= \"English\",\n        licence=\"DbCL v1.0\",\n        default_target_attribute=\"label\",\n        attributes=\"auto\",\n        data=df,\n        citation=citation,\n        ignore_attribute=None\n    )\n    openml.config.apikey = ''\n    tinyim.publish()\n    print(f\"URL for dataset: {tinyim.openml_url}\")\n</pre> def create_tiny_imagenet():     dir_name = \"datasets\"     os.makedirs(dir_name, exist_ok=True)      # download the dataset     url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"     r = requests.get(url, stream=True)      if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):         with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:             f.write(r.content)          with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:             zip_ref.extractall(f\"{dir_name}/\")     ## recusively find all the images     image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")     ## remove the first part of the path     image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]     ## create a dataframe with the image path and the label     label_func = lambda x: x.split(\"/\")[2]     df = pd.DataFrame(image_paths, columns=[\"image_path\"])     df[\"label\"] = df[\"image_path\"].apply(label_func)     ## encode the labels as integers     # df[\"Class_encoded\"] = pd.factorize(df[\"label\"])[0]      ## encode types     df[\"image_path\"] = df[\"image_path\"].astype(\"string\")     df[\"label\"] = df[\"label\"].astype(\"string\")       name = \"tiny-imagenet-200\"     attribute_names = df.columns     description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"     paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"     citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")      tinyim = create_dataset(         name = name,         description = description,         creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         collection_date = \"2017\",         language= \"English\",         licence=\"DbCL v1.0\",         default_target_attribute=\"label\",         attributes=\"auto\",         data=df,         citation=citation,         ignore_attribute=None     )     openml.config.apikey = ''     tinyim.publish()     print(f\"URL for dataset: {tinyim.openml_url}\")  In\u00a0[10]: Copied! <pre>create_tiny_imagenet()\n# https://www.openml.org/d/46346\n</pre> create_tiny_imagenet() # https://www.openml.org/d/46346 In\u00a0[19]: Copied! <pre>def create_tiniest_imagenet():\n    dir_name = \"datasets\"\n    os.makedirs(dir_name, exist_ok=True)\n\n    # download the dataset\n    url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"\n    r = requests.get(url, stream=True)\n\n    if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):\n        with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:\n            f.write(r.content)\n\n        with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:\n            zip_ref.extractall(f\"{dir_name}/\")\n    ## recusively find all the images\n    image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")\n    ## remove the first part of the path\n    image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]\n    image_paths[-1]\n    ## create a dataframe with the image path and the label\n    label_func = lambda x: x.split(\"/\")[2]\n    df = pd.DataFrame(image_paths, columns=[\"image_path\"])\n    df[\"label\"] = df[\"image_path\"].apply(label_func)\n    ## encode types\n    df[\"image_path\"] = df[\"image_path\"].astype(\"string\")\n    df[\"label\"] = df[\"label\"].astype(\"string\")\n\n    # keep only first 20 images for each label\n    df = df.groupby(\"label\").head(20)\n\n\n    name = \"tiniest-imagenet-200\"\n    attribute_names = df.columns\n    description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. !!! This dataset only links to 20 images per class (instead of the usual 500) and is ONLY for quickly testing a framework. !!! Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"\n    paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"\n    citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")\n\n    tinyim = create_dataset(\n        name = name,\n        description = description,\n        creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        collection_date = \"2017\",\n        language= \"English\",\n        licence=\"DbCL v1.0\",\n        default_target_attribute=\"label\",\n        attributes=\"auto\",\n        data=df,\n        citation=citation,\n        ignore_attribute=None\n    )\n    openml.config.apikey = ''\n    tinyim.publish()\n    print(f\"URL for dataset: {tinyim.openml_url}\")\n</pre> def create_tiniest_imagenet():     dir_name = \"datasets\"     os.makedirs(dir_name, exist_ok=True)      # download the dataset     url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"     r = requests.get(url, stream=True)      if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):         with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:             f.write(r.content)          with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:             zip_ref.extractall(f\"{dir_name}/\")     ## recusively find all the images     image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")     ## remove the first part of the path     image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]     image_paths[-1]     ## create a dataframe with the image path and the label     label_func = lambda x: x.split(\"/\")[2]     df = pd.DataFrame(image_paths, columns=[\"image_path\"])     df[\"label\"] = df[\"image_path\"].apply(label_func)     ## encode types     df[\"image_path\"] = df[\"image_path\"].astype(\"string\")     df[\"label\"] = df[\"label\"].astype(\"string\")      # keep only first 20 images for each label     df = df.groupby(\"label\").head(20)       name = \"tiniest-imagenet-200\"     attribute_names = df.columns     description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. !!! This dataset only links to 20 images per class (instead of the usual 500) and is ONLY for quickly testing a framework. !!! Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"     paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"     citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")      tinyim = create_dataset(         name = name,         description = description,         creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         collection_date = \"2017\",         language= \"English\",         licence=\"DbCL v1.0\",         default_target_attribute=\"label\",         attributes=\"auto\",         data=df,         citation=citation,         ignore_attribute=None     )     openml.config.apikey = ''     tinyim.publish()     print(f\"URL for dataset: {tinyim.openml_url}\")  In\u00a0[20]: Copied! <pre>create_tiniest_imagenet()\n# https://www.openml.org/d/46347\n</pre> create_tiniest_imagenet() # https://www.openml.org/d/46347 <pre>URL for dataset: https://www.openml.org/d/46347\n</pre> In\u00a0[27]: Copied! <pre>def create_task():\n    # Define task parameters\n    task_type = openml.tasks.TaskType.SUPERVISED_CLASSIFICATION\n    dataset_id = 46347 # Obtained from the dataset creation step\n    evaluation_measure = 'predictive_accuracy'\n    target_name = 'label'\n    class_labels = list(pd.read_csv(\"datasets/tiniest_imagenet.csv\")[\"label\"].unique())\n    cost_matrix = None\n\n    # Create the task\n    new_task = openml.tasks.create_task(\n        task_type=task_type,\n        dataset_id=dataset_id, \n        estimation_procedure_id = 1,\n        evaluation_measure=evaluation_measure,\n        target_name=target_name,\n        class_labels=class_labels,\n        cost_matrix=cost_matrix\n    )\n    openml.config.apikey = ''\n    new_task.publish()\n    print(f\"URL for task: {new_task.openml_url}\")\n</pre> def create_task():     # Define task parameters     task_type = openml.tasks.TaskType.SUPERVISED_CLASSIFICATION     dataset_id = 46347 # Obtained from the dataset creation step     evaluation_measure = 'predictive_accuracy'     target_name = 'label'     class_labels = list(pd.read_csv(\"datasets/tiniest_imagenet.csv\")[\"label\"].unique())     cost_matrix = None      # Create the task     new_task = openml.tasks.create_task(         task_type=task_type,         dataset_id=dataset_id,          estimation_procedure_id = 1,         evaluation_measure=evaluation_measure,         target_name=target_name,         class_labels=class_labels,         cost_matrix=cost_matrix     )     openml.config.apikey = ''     new_task.publish()     print(f\"URL for task: {new_task.openml_url}\") In\u00a0[28]: Copied! <pre>create_task()\n# https://www.openml.org/t/362128\n</pre> create_task() # https://www.openml.org/t/362128 <pre>URL for task: https://www.openml.org/t/362128\n</pre>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#create-dataset-and-task-tiniest-imagenet","title":"Create dataset and task - tiniest imagenet\u00b6","text":"<ul> <li>An example of how to create a custom dataset and task using the OpenML API and upload it to the OpenML server.</li> <li>Note that you must have an API key from the OpenML website to upload datasets and tasks.</li> </ul>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#create-dataset-on-openml","title":"Create dataset on OpenML\u00b6","text":"<ul> <li>Instead of making our own, we obtain a subset of the ImageNet dataset from Stanford. This dataset has 200 classes.</li> </ul>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#another-even-tinier-dataset","title":"Another, even tinier dataset\u00b6","text":"<ul> <li>We subset the previous dataset to 20 images per class.</li> </ul>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#create-task-on-openml","title":"Create task on OpenML\u00b6","text":"<ul> <li>Now to actually use the OpenML Pytorch API, we need to have a task associated with the dataset. This is how we create it.</li> </ul>"},{"location":"pytorch/Examples/Image%20Classification%20Task/","title":"Image classification task","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml_pytorch.config\nimport openml\nimport logging\nimport warnings\n\n# Suppress FutureWarning messages\nwarnings.simplefilter(action='ignore')\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n\n############################################################################\nfrom openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda\nimport torchvision\n\nfrom openml_pytorch.trainer import convert_to_rgb\n</pre> import torch.nn import torch.optim  import openml_pytorch.config import openml import logging import warnings  # Suppress FutureWarning messages warnings.simplefilter(action='ignore')  ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################  ############################################################################ from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule from torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda import torchvision  from openml_pytorch.trainer import convert_to_rgb In\u00a0[\u00a0]: Copied! <pre>model = torchvision.models.efficientnet_b0(num_classes=200)\n</pre> model = torchvision.models.efficientnet_b0(num_classes=200) In\u00a0[\u00a0]: Copied! <pre>transform = Compose(\n    [\n        ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n        Lambda(\n            convert_to_rgb\n        ),  # Convert PIL Image to RGB if it's not already.\n        Resize(\n            (64, 64)\n        ),  # Resize the image.\n        ToTensor(),  # Convert the PIL Image back to a tensor.\n    ]\n)\ndata_module = OpenMLDataModule(\n    type_of_data=\"image\",\n    file_dir=\"datasets\",\n    filename_col=\"image_path\",\n    target_mode=\"categorical\",\n    target_column=\"label\",\n    batch_size = 64,\n    transform=transform\n)\n</pre> transform = Compose(     [         ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.         Lambda(             convert_to_rgb         ),  # Convert PIL Image to RGB if it's not already.         Resize(             (64, 64)         ),  # Resize the image.         ToTensor(),  # Convert the PIL Image back to a tensor.     ] ) data_module = OpenMLDataModule(     type_of_data=\"image\",     file_dir=\"datasets\",     filename_col=\"image_path\",     target_mode=\"categorical\",     target_column=\"label\",     batch_size = 64,     transform=transform ) In\u00a0[\u00a0]: Copied! <pre>trainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n    callbacks=[],\n)\nopenml_pytorch.config.trainer = trainer\n</pre> trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 1,     callbacks=[], ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre># Download the OpenML task for tiniest imagenet\ntask = openml.tasks.get_task(362128)\n</pre> # Download the OpenML task for tiniest imagenet task = openml.tasks.get_task(362128) In\u00a0[\u00a0]: Copied! <pre>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr() In\u00a0[\u00a0]: Copied! <pre>trainer.learn.model_classes\n</pre> trainer.learn.model_classes In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish()"},{"location":"pytorch/Examples/Image%20Classification%20Task/#image-classification-task","title":"Image classification task\u00b6","text":"<ul> <li>Image classification on OpenML Task (362128), tiniest ImageNet dataset.</li> </ul>"},{"location":"pytorch/Examples/Image%20Classification%20Task/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the data is present in the <code>file_dir</code> directory, and the <code>filename_col</code> is correctly set along with this column correctly pointing to where your data is stored.</li> </ul>"},{"location":"pytorch/Examples/Image%20Classification%20Task/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#download-the-task","title":"Download the task\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#view-the-classes-in-the-model","title":"View the classes in the model\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#publish-the-run-to-openml","title":"Publish the run to OpenML\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/","title":"Pretrained Image classification example - Transformer","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml\nimport openml_pytorch\nimport openml_pytorch.layers\nimport openml_pytorch.config\nfrom openml import OpenMLTask\nimport logging\nimport warnings\nfrom torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda\nfrom openml_pytorch.trainer import convert_to_rgb\n# Suppress FutureWarning messages\nwarnings.simplefilter(action='ignore')\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n\n############################################################################\nimport torch.nn as nn\nimport torch.nn.functional as F\n</pre> import torch.nn import torch.optim  import openml import openml_pytorch import openml_pytorch.layers import openml_pytorch.config from openml import OpenMLTask import logging import warnings from torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda from openml_pytorch.trainer import convert_to_rgb # Suppress FutureWarning messages warnings.simplefilter(action='ignore')  ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################  ############################################################################ import torch.nn as nn import torch.nn.functional as F In\u00a0[\u00a0]: Copied! <pre># openml.config.apikey = 'key'\nfrom openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom openml_pytorch.trainer import Callback\n</pre> # openml.config.apikey = 'key' from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule from openml_pytorch.trainer import Callback In\u00a0[\u00a0]: Copied! <pre># Example model. You can do better :)\nimport torchvision.models as models\n\n# Load the pre-trained ResNet model\nmodel = models.efficientnet_b0(pretrained=True)\n\n# Modify the last fully connected layer to the required number of classes\nnum_classes = 200\nin_features = model.classifier[-1].in_features\n# model.fc = nn.Linear(in_features, num_classes)\nmodel.classifier = nn.Sequential(\n    nn.Dropout(p=0.2, inplace=True),\n    nn.Linear(in_features, num_classes),\n)\n\n# Optional: If you're fine-tuning, you may want to freeze the pre-trained layers\n# for param in model.parameters():\n#     param.requires_grad = False\n\n# # If you want to train the last layer only (the newly added layer)\n# for param in model.fc.parameters():\n#     param.requires_grad = True\n</pre> # Example model. You can do better :) import torchvision.models as models  # Load the pre-trained ResNet model model = models.efficientnet_b0(pretrained=True)  # Modify the last fully connected layer to the required number of classes num_classes = 200 in_features = model.classifier[-1].in_features # model.fc = nn.Linear(in_features, num_classes) model.classifier = nn.Sequential(     nn.Dropout(p=0.2, inplace=True),     nn.Linear(in_features, num_classes), )  # Optional: If you're fine-tuning, you may want to freeze the pre-trained layers # for param in model.parameters(): #     param.requires_grad = False  # # If you want to train the last layer only (the newly added layer) # for param in model.fc.parameters(): #     param.requires_grad = True In\u00a0[\u00a0]: Copied! <pre>transform = Compose(\n    [\n        ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n        Lambda(\n            convert_to_rgb\n        ),  # Convert PIL Image to RGB if it's not already.\n        Resize(\n            (64, 64)\n        ),  # Resize the image.\n        ToTensor(),  # Convert the PIL Image back to a tensor.\n    ]\n)\ndata_module = OpenMLDataModule(\n    type_of_data=\"image\",\n    file_dir=\"datasets\",\n    filename_col=\"image_path\",\n    target_mode=\"categorical\",\n    target_column=\"label\",\n    batch_size = 64,\n    transform=transform\n)\n</pre> transform = Compose(     [         ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.         Lambda(             convert_to_rgb         ),  # Convert PIL Image to RGB if it's not already.         Resize(             (64, 64)         ),  # Resize the image.         ToTensor(),  # Convert the PIL Image back to a tensor.     ] ) data_module = OpenMLDataModule(     type_of_data=\"image\",     file_dir=\"datasets\",     filename_col=\"image_path\",     target_mode=\"categorical\",     target_column=\"label\",     batch_size = 64,     transform=transform ) In\u00a0[\u00a0]: Copied! <pre>def custom_optimizer_gen(model: torch.nn.Module, task: OpenMLTask) -&gt; torch.optim.Optimizer:\n    return torch.optim.Adam(model.fc.parameters())\n\ntrainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n    optimizer = custom_optimizer_gen,\n    callbacks=[],\n)\nopenml_pytorch.config.trainer = trainer\n</pre> def custom_optimizer_gen(model: torch.nn.Module, task: OpenMLTask) -&gt; torch.optim.Optimizer:     return torch.optim.Adam(model.fc.parameters())  trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 1,     optimizer = custom_optimizer_gen,     callbacks=[], ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre># Download the OpenML task for tiniest imagenet\ntask = openml.tasks.get_task(362128)\n</pre>  # Download the OpenML task for tiniest imagenet task = openml.tasks.get_task(362128) In\u00a0[\u00a0]: Copied! <pre>#\n# Run the model on the task (requires an API key).m\nrun = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> # # Run the model on the task (requires an API key).m run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr() In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish()"},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#pretrained-image-classification-example-transformer","title":"Pretrained Image classification example - Transformer\u00b6","text":"<ul> <li>Pretrained image classification using a Transformer architecture, \"custom\" Optimizer for OpenML Task (362128) , tiniest ImageNet dataset.</li> </ul>"},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the data is present in the <code>file_dir</code> directory, and the <code>filename_col</code> is correctly set along with this column correctly pointing to where your data is stored.</li> </ul>"},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#download-the-task","title":"Download the task\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#publish-the-run-to-openml","title":"Publish the run to OpenML\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/","title":"Sequential classification","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml_pytorch.config\nimport openml\nimport logging\nimport warnings\n\n# Suppress FutureWarning messages\nwarnings.simplefilter(action='ignore')\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n</pre>  import torch.nn import torch.optim  import openml_pytorch.config import openml import logging import warnings  # Suppress FutureWarning messages warnings.simplefilter(action='ignore')  ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################ In\u00a0[\u00a0]: Copied! <pre>from openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\n</pre> from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule In\u00a0[\u00a0]: Copied! <pre>############################################################################\n# Define a sequential network that does the initial image reshaping\n# and normalization model.\nprocessing_net = torch.nn.Sequential(\n    openml_pytorch.layers.Functional(function=torch.Tensor.reshape,\n                                                shape=(-1, 1, 28, 28)),\n    torch.nn.BatchNorm2d(num_features=1)\n)\n############################################################################\n\n############################################################################\n# Define a sequential network that does the extracts the features from the\n# image.\nfeatures_net = torch.nn.Sequential(\n    torch.nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5),\n    torch.nn.LeakyReLU(),\n    torch.nn.MaxPool2d(kernel_size=2),\n    torch.nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5),\n    torch.nn.LeakyReLU(),\n    torch.nn.MaxPool2d(kernel_size=2),\n)\n############################################################################\n\n############################################################################\n# Define a sequential network that flattens the features and compiles the\n# results into probabilities for each digit.\nresults_net = torch.nn.Sequential(\n    openml_pytorch.layers.Functional(function=torch.Tensor.reshape,\n                                                shape=(-1, 4 * 4 * 64)),\n    torch.nn.Linear(in_features=4 * 4 * 64, out_features=256),\n    torch.nn.LeakyReLU(),\n    torch.nn.Dropout(),\n    torch.nn.Linear(in_features=256, out_features=10),\n)\n############################################################################\n# openml.config.apikey = 'key'\n\n############################################################################\n# The main network, composed of the above specified networks.\nmodel = torch.nn.Sequential(\n    processing_net,\n    features_net,\n    results_net\n)\n############################################################################\n</pre>  ############################################################################ # Define a sequential network that does the initial image reshaping # and normalization model. processing_net = torch.nn.Sequential(     openml_pytorch.layers.Functional(function=torch.Tensor.reshape,                                                 shape=(-1, 1, 28, 28)),     torch.nn.BatchNorm2d(num_features=1) ) ############################################################################  ############################################################################ # Define a sequential network that does the extracts the features from the # image. features_net = torch.nn.Sequential(     torch.nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5),     torch.nn.LeakyReLU(),     torch.nn.MaxPool2d(kernel_size=2),     torch.nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5),     torch.nn.LeakyReLU(),     torch.nn.MaxPool2d(kernel_size=2), ) ############################################################################  ############################################################################ # Define a sequential network that flattens the features and compiles the # results into probabilities for each digit. results_net = torch.nn.Sequential(     openml_pytorch.layers.Functional(function=torch.Tensor.reshape,                                                 shape=(-1, 4 * 4 * 64)),     torch.nn.Linear(in_features=4 * 4 * 64, out_features=256),     torch.nn.LeakyReLU(),     torch.nn.Dropout(),     torch.nn.Linear(in_features=256, out_features=10), ) ############################################################################ # openml.config.apikey = 'key'  ############################################################################ # The main network, composed of the above specified networks. model = torch.nn.Sequential(     processing_net,     features_net,     results_net ) ############################################################################  In\u00a0[\u00a0]: Copied! <pre>data_module = OpenMLDataModule(\n    type_of_data=\"dataframe\",\n    filename_col=\"class\",\n    target_mode=\"categorical\",\n)\n</pre> data_module = OpenMLDataModule(     type_of_data=\"dataframe\",     filename_col=\"class\",     target_mode=\"categorical\", ) In\u00a0[\u00a0]: Copied! <pre>trainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n    callbacks=[],\n)\nopenml_pytorch.config.trainer = trainer\n</pre>  trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 1,     callbacks=[], ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre># Download the OpenML task for the mnist 784 dataset.\ntask = openml.tasks.get_task(3573)\n</pre> # Download the OpenML task for the mnist 784 dataset. task = openml.tasks.get_task(3573) In\u00a0[\u00a0]: Copied! <pre>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr() In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish()"},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#sequential-classification","title":"Sequential classification\u00b6","text":"<ul> <li>Sequential classification of a tabular MNIST dataset (Task 3573) using a simple neural network.</li> </ul>"},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the <code>target_col</code> is correctly set.</li> </ul>"},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#download-the-task","title":"Download the task\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#publish-the-run-to-openml","title":"Publish the run to OpenML\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/","title":"Tabular classification","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml\nimport openml_pytorch\nimport openml_pytorch.layers\nimport openml_pytorch.config\nimport logging\n\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n</pre>  import torch.nn import torch.optim  import openml import openml_pytorch import openml_pytorch.layers import openml_pytorch.config import logging   ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################ In\u00a0[\u00a0]: Copied! <pre>from openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom openml_pytorch.trainer import Callback\n</pre> from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule from openml_pytorch.trainer import Callback In\u00a0[\u00a0]: Copied! <pre>class TabularClassificationmodel(torch.nn.Module):\n    def __init__(self, input_size, output_size):\n        super(TabularClassificationmodel, self).__init__()\n        self.fc1 = torch.nn.Linear(input_size, 128)\n        self.fc2 = torch.nn.Linear(128, 64)\n        self.fc3 = torch.nn.Linear(64, output_size)\n        self.relu = torch.nn.ReLU()\n        self.softmax = torch.nn.Softmax(dim=1)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.relu(x)\n        x = self.fc3(x)\n        x = self.softmax(x)\n        return x\n</pre> class TabularClassificationmodel(torch.nn.Module):     def __init__(self, input_size, output_size):         super(TabularClassificationmodel, self).__init__()         self.fc1 = torch.nn.Linear(input_size, 128)         self.fc2 = torch.nn.Linear(128, 64)         self.fc3 = torch.nn.Linear(64, output_size)         self.relu = torch.nn.ReLU()         self.softmax = torch.nn.Softmax(dim=1)      def forward(self, x):         x = self.fc1(x)         x = self.relu(x)         x = self.fc2(x)         x = self.relu(x)         x = self.fc3(x)         x = self.softmax(x)         return x In\u00a0[\u00a0]: Copied! <pre>model = TabularClassificationmodel(20, 2)\n</pre> model = TabularClassificationmodel(20, 2) In\u00a0[\u00a0]: Copied! <pre># supervised credit-g classification\ntask = openml.tasks.get_task(31)\n</pre> # supervised credit-g classification task = openml.tasks.get_task(31) In\u00a0[\u00a0]: Copied! <pre>data_module = OpenMLDataModule(\n    type_of_data=\"dataframe\",\n    target_column=\"class\",\n    target_mode=\"categorical\",\n)\n</pre> data_module = OpenMLDataModule(     type_of_data=\"dataframe\",     target_column=\"class\",     target_mode=\"categorical\", ) In\u00a0[\u00a0]: Copied! <pre>trainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 5,\n)\nopenml_pytorch.config.trainer = trainer\n</pre>  trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 5, ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish() In\u00a0[\u00a0]: Copied! <pre># openml.config.apikey = ''\n</pre> # openml.config.apikey = '' In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr()"},{"location":"pytorch/Examples/Tabular%20Classification/#tabular-classification","title":"Tabular classification\u00b6","text":"<ul> <li>Supervised credit-g classification</li> </ul>"},{"location":"pytorch/Examples/Tabular%20Classification/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the <code>target_col</code> is correctly set.</li> </ul>"},{"location":"pytorch/Examples/Tabular%20Classification/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"r/","title":"mlr3oml","text":"<p>Package website: release | dev</p> <p>OpenML integration to the mlr3 ecosystem.</p> <p> </p>"},{"location":"r/#what-is-mlr3oml","title":"What is <code>mlr3oml</code>?","text":"<p>OpenML is an open-source platform that facilitates the sharing and dissemination of machine learning research data. All entities on the platform have unique identifiers and standardized (meta)data that can be accessed via an open-access REST API or the web interface. <code>mlr3oml</code> allows to work with the REST API through R and integrates OpenML with the <code>mlr3</code> ecosystem. Note that some upload options are currently not supported, use the OpenML package package for this.</p> <p>As a brief demo, we show how to access an OpenML task, convert it to an <code>mlr3::Task</code> and associated <code>mlr3::Resampling</code>, and conduct a simple resample experiment.</p> <pre><code>library(mlr3oml)\nlibrary(mlr3)\n\n# Download and print the OpenML task with ID 145953\noml_task = otsk(145953)\noml_task\n</code></pre> <pre><code>## &lt;OMLTask:145953&gt;\n##  * Type: Supervised Classification\n##  * Data: kr-vs-kp (id: 3; dim: 3196x37)\n##  * Target: class\n##  * Estimation: crossvalidation (id: 1; repeats: 1, folds: 10)\n</code></pre> <pre><code># Access the OpenML data object on which the task is built\noml_task$data\n</code></pre> <pre><code>## &lt;OMLData:3:kr-vs-kp&gt; (3196x37)\n##  * Default target: class\n</code></pre> <pre><code># Convert the OpenML task to an mlr3 task and resampling\ntask = as_task(oml_task)\nresampling = as_resampling(oml_task)\n\n# Conduct a simple resample experiment\nrr = resample(task, lrn(\"classif.rpart\"), resampling)\nrr$aggregate()\n</code></pre> <pre><code>## classif.ce \n##  0.0319181\n</code></pre> <p>Besides working with objects with known IDs, data of interest can also be queried using listing functions. Below, we search for datasets with 10 - 20 features, 100 to 10000 observations and 2 classes.</p> <pre><code>odatasets = list_oml_data(\n  number_features = c(10, 20),\n  number_instances = c(100, 10000),\n  number_classes = 2\n)\n\nhead(odatasets[, c(\"data_id\", \"name\")])\n</code></pre> <pre><code>##    data_id            name\n## 1:      13   breast-cancer\n## 2:      15        breast-w\n## 3:      29 credit-approval\n## 4:      49         heart-c\n## 5:      50     tic-tac-toe\n## 6:      51         heart-h\n</code></pre> <p>To retrieve individual datasets, you can use <code>odt</code> and either manually construct a new <code>Task</code> object using <code>as_task()</code> or use it <code>data.table</code> format.</p> <pre><code>odataset = odt(29)\n\n# Dataset as data.table\nstr(odataset$data)\n</code></pre> <pre><code>## Classes 'data.table' and 'data.frame':   690 obs. of  16 variables:\n##  $ A1   : Factor w/ 2 levels \"b\",\"a\": 1 2 2 1 1 1 1 2 1 1 ...\n##  $ A2   : num  30.8 58.7 24.5 27.8 20.2 ...\n##  $ A3   : num  0 4.46 0.5 1.54 5.62 ...\n##  $ A4   : Factor w/ 4 levels \"u\",\"y\",\"l\",\"t\": 1 1 1 1 1 1 1 1 2 2 ...\n##  $ A5   : Factor w/ 3 levels \"g\",\"p\",\"gg\": 1 1 1 1 1 1 1 1 2 2 ...\n##  $ A6   : Factor w/ 14 levels \"c\",\"d\",\"cc\",\"i\",..: 10 9 9 10 10 7 8 3 6 10 ...\n##  $ A7   : Factor w/ 9 levels \"v\",\"h\",\"bb\",\"j\",..: 1 2 2 1 1 1 2 1 2 1 ...\n##  $ A8   : num  1.25 3.04 1.5 3.75 1.71 ...\n##  $ A9   : Factor w/ 2 levels \"t\",\"f\": 1 1 1 1 1 1 1 1 1 1 ...\n##  $ A10  : Factor w/ 2 levels \"t\",\"f\": 1 1 2 1 2 2 2 2 2 2 ...\n##  $ A11  : int  1 6 0 5 0 0 0 0 0 0 ...\n##  $ A12  : Factor w/ 2 levels \"t\",\"f\": 2 2 2 1 2 1 1 2 2 1 ...\n##  $ A13  : Factor w/ 3 levels \"g\",\"p\",\"s\": 1 1 1 1 3 1 1 1 1 1 ...\n##  $ A14  : int  202 43 280 100 120 360 164 80 180 52 ...\n##  $ A15  : int  0 560 824 3 0 0 31285 1349 314 1442 ...\n##  $ class: Factor w/ 2 levels \"+\",\"-\": 1 1 1 1 1 1 1 1 1 1 ...\n##  - attr(*, \".internal.selfref\")=&lt;externalptr&gt;\n</code></pre> <pre><code># Creating a new task\notask = as_task(odataset)\notask\n</code></pre> <pre><code>## &lt;TaskClassif:credit-approval&gt; (690 x 16)\n## * Target: class\n## * Properties: twoclass\n## * Features (15):\n##   - fct (9): A1, A10, A12, A13, A4, A5, A6, A7, A9\n##   - int (3): A11, A14, A15\n##   - dbl (3): A2, A3, A8\n</code></pre>"},{"location":"r/#feature-overview","title":"Feature Overview","text":"<ul> <li>Datasets, tasks, flows, runs, and collections can be downloaded from   OpenML and are represented as <code>R6</code> classes.</li> <li>OpenML objects can be easily converted to the corresponding <code>mlr3</code>   counterpart.</li> <li>Filtering of OpenML objects can be achieved using listing functions.</li> <li>Downloaded objects can be cached by setting the <code>mlr3oml.cache</code>   option.</li> <li>Both the <code>arff</code> and <code>parquet</code> filetype for datasets are supported.</li> <li>You can upload datasets, tasks, and collections to OpenML.</li> </ul>"},{"location":"r/#documentation","title":"Documentation","text":"<ul> <li>Start by reading the Large-Scale Benchmarking   chapter   from the <code>mlr3</code> book.</li> <li>The package website contains a   getting started guide.</li> <li>The OpenML API documentation is also a   good resource.</li> </ul>"},{"location":"r/#bugs-questions-feedback","title":"Bugs, Questions, Feedback","text":"<p>mlr3oml is a free and open source software project that encourages participation and feedback. If you have any issues, questions, suggestions or feedback, please do not hesitate to open an \u201cissue\u201d about it on the GitHub page!</p> <p>In case of problems / bugs, it is often helpful if you provide a \u201cminimum working example\u201d that showcases the behaviour (but don\u2019t worry about this if the bug is obvious).</p>"},{"location":"reference/","title":"Index","text":"<p>The OpenML module implements a python interface to <code>OpenML &lt;https://www.openml.org&gt;</code>_, a collaborative platform for machine learning. OpenML can be used to</p> <ul> <li>store, download and analyze datasets</li> <li>make experiments and their results (e.g. models, predictions)   accesible and reproducible for everybody</li> <li>analyze experiments (uploaded by you and other collaborators) and conduct   meta studies</li> </ul> <p>In particular, this module implements a python interface for the <code>OpenML REST API &lt;https://www.openml.org/guide#!rest_services&gt;</code>_ (<code>REST on wikipedia &lt;https://en.wikipedia.org/wiki/Representational_state_transfer&gt;</code>_).</p>"},{"location":"reference/#openml.OpenMLBenchmarkSuite","title":"<code>OpenMLBenchmarkSuite</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).</p> <p>It contains the following information: name, id, description, creation date, creator id and the task ids.</p> <p>According to this list of task ids, the suite object receives a list of OpenML object ids (datasets).</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLBenchmarkSuite(BaseStudy):\n    \"\"\"\n    An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and the task ids.\n\n    According to this list of task ids, the suite object receives a list of\n    OpenML object ids (datasets).\n\n    Parameters\n    ----------\n    suite_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        suite_id: int | None,\n        alias: str | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n    ):\n        super().__init__(\n            study_id=suite_id,\n            alias=alias,\n            main_entity_type=\"task\",\n            benchmark_suite=None,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=None,\n            runs=None,\n            setups=None,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLClassificationTask","title":"<code>OpenMLClassificationTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Classification task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the Classification task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the Classification task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target variable.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the Classification task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the Classification task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the Classification task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Classification task (if it already exists on OpenML).</p> <code>None</code> <code>class_labels</code> <code>List of str</code> <p>A list of class labels (for classification tasks).</p> <code>None</code> <code>cost_matrix</code> <code>array</code> <p>A cost matrix (for classification tasks).</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClassificationTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Classification task type.\n    task_type : str\n        Name of the Classification task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the Classification task.\n    target_name : str\n        Name of the target variable.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the Classification task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the Classification task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure.\n    data_splits_url : str, default=None\n        URL of the data splits for the Classification task.\n    task_id : Union[int, None]\n        ID of the Classification task (if it already exists on OpenML).\n    class_labels : List of str, default=None\n        A list of class labels (for classification tasks).\n    cost_matrix : array, default=None\n        A cost matrix (for classification tasks).\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n        self.class_labels = class_labels\n        self.cost_matrix = cost_matrix\n\n        if cost_matrix is not None:\n            raise NotImplementedError(\"Costmatrix\")\n</code></pre>"},{"location":"reference/#openml.OpenMLClusteringTask","title":"<code>OpenMLClusteringTask</code>","text":"<p>               Bases: <code>OpenMLTask</code></p> <p>OpenML Clustering object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML clustering task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML clustering task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset used in clustering the task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML clustering task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure used in the clustering task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the clustering task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the clustering task.</p> <code>None</code> <code>target_name</code> <code>str</code> <p>Name of the target feature (class) that is not part of the feature set for the clustering task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClusteringTask(OpenMLTask):\n    \"\"\"OpenML Clustering object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML clustering task.\n    task_type : str\n        Task type of the OpenML clustering task.\n    data_set_id : int\n        ID of the OpenML dataset used in clustering the task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    task_id : Union[int, None]\n        ID of the OpenML clustering task.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure used in the clustering task.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the clustering task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the clustering task.\n    target_name : str, default=None\n        Name of the target feature (class) that is not part of the\n        feature set for the clustering task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 17,\n        task_id: int | None = None,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        evaluation_measure: str | None = None,\n        target_name: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            evaluation_measure=evaluation_measure,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\"] = \"array\",\n    ) -&gt; np.ndarray | scipy.sparse.spmatrix:\n        ...\n\n    @overload\n    def get_X(self, dataset_format: Literal[\"dataframe\"]) -&gt; pd.DataFrame:\n        ...\n\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n    ) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        dataset = self.get_dataset()\n        data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n        return data\n\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        # Right now, it is not supported as a feature.\n        # Uncomment if it is supported on the server\n        # in the future.\n        # https://github.com/openml/OpenML/issues/925\n        \"\"\"\n        task_dict = task_container['oml:task_inputs']\n        if self.target_name is not None:\n            task_dict['oml:input'].append(\n                OrderedDict([\n                    ('@name', 'target_feature'),\n                    ('#text', self.target_name)\n                ])\n            )\n        \"\"\"\n        return super()._to_dict()\n</code></pre>"},{"location":"reference/#openml.OpenMLClusteringTask.get_X","title":"<code>get_X(dataset_format='array')</code>","text":"<pre><code>get_X(dataset_format: Literal['array'] = 'array') -&gt; np.ndarray | scipy.sparse.spmatrix\n</code></pre><pre><code>get_X(dataset_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X(\n    self,\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    dataset = self.get_dataset()\n    data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n    return data\n</code></pre>"},{"location":"reference/#openml.OpenMLDataFeature","title":"<code>OpenMLDataFeature</code>","text":"<p>Data Feature (a.k.a. Attribute) object.</p> <p>Parameters:</p> Name Type Description Default <code>index</code> <code>int</code> <p>The index of this feature</p> required <code>name</code> <code>str</code> <p>Name of the feature</p> required <code>data_type</code> <code>str</code> <p>can be nominal, numeric, string, date (corresponds to arff)</p> required <code>nominal_values</code> <code>list(str)</code> <p>list of the possible values, in case of nominal attribute</p> required <code>number_missing_values</code> <code>int</code> <p>Number of rows that have a missing value for this feature.</p> required <code>ontologies</code> <code>list(str)</code> <p>list of ontologies attached to this feature. An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided.</p> <code>None</code> Source code in <code>openml/datasets/data_feature.py</code> <pre><code>class OpenMLDataFeature:\n    \"\"\"\n    Data Feature (a.k.a. Attribute) object.\n\n    Parameters\n    ----------\n    index : int\n        The index of this feature\n    name : str\n        Name of the feature\n    data_type : str\n        can be nominal, numeric, string, date (corresponds to arff)\n    nominal_values : list(str)\n        list of the possible values, in case of nominal attribute\n    number_missing_values : int\n        Number of rows that have a missing value for this feature.\n    ontologies : list(str)\n        list of ontologies attached to this feature. An ontology describes the\n        concept that are described in a feature. An ontology is defined by an\n        URL where the information is provided.\n    \"\"\"\n\n    LEGAL_DATA_TYPES: ClassVar[Sequence[str]] = [\"nominal\", \"numeric\", \"string\", \"date\"]\n\n    def __init__(  # noqa: PLR0913\n        self,\n        index: int,\n        name: str,\n        data_type: str,\n        nominal_values: list[str],\n        number_missing_values: int,\n        ontologies: list[str] | None = None,\n    ):\n        if not isinstance(index, int):\n            raise TypeError(f\"Index must be `int` but is {type(index)}\")\n\n        if data_type not in self.LEGAL_DATA_TYPES:\n            raise ValueError(\n                f\"data type should be in {self.LEGAL_DATA_TYPES!s}, found: {data_type}\",\n            )\n\n        if data_type == \"nominal\":\n            if nominal_values is None:\n                raise TypeError(\n                    \"Dataset features require attribute `nominal_values` for nominal \"\n                    \"feature type.\",\n                )\n\n            if not isinstance(nominal_values, list):\n                raise TypeError(\n                    \"Argument `nominal_values` is of wrong datatype, should be list, \"\n                    f\"but is {type(nominal_values)}\",\n                )\n        elif nominal_values is not None:\n            raise TypeError(\"Argument `nominal_values` must be None for non-nominal feature.\")\n\n        if not isinstance(number_missing_values, int):\n            msg = f\"number_missing_values must be int but is {type(number_missing_values)}\"\n            raise TypeError(msg)\n\n        self.index = index\n        self.name = str(name)\n        self.data_type = str(data_type)\n        self.nominal_values = nominal_values\n        self.number_missing_values = number_missing_values\n        self.ontologies = ontologies\n\n    def __repr__(self) -&gt; str:\n        return \"[%d - %s (%s)]\" % (self.index, self.name, self.data_type)\n\n    def __eq__(self, other: Any) -&gt; bool:\n        return isinstance(other, OpenMLDataFeature) and self.__dict__ == other.__dict__\n\n    def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -&gt; None:  # noqa: FBT001, ARG002\n        pp.text(str(self))\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset","title":"<code>OpenMLDataset</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>Dataset object.</p> <p>Allows fetching and uploading datasets to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>data_format</code> <code>str</code> <p>Format of the dataset which can be either 'arff' or 'sparse_arff'.</p> <code>'arff'</code> <code>cache_format</code> <code>str</code> <p>Format for caching the dataset which can be either 'feather' or 'pickle'.</p> <code>'pickle'</code> <code>dataset_id</code> <code>int</code> <p>Id autogenerated by the server.</p> <code>None</code> <code>version</code> <code>int</code> <p>Version of this dataset. '1' for original version. Auto-incremented by server.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>The date-time when the dataset was uploaded, generated by server.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>licence</code> <code>str</code> <p>License of the data.</p> <code>None</code> <code>url</code> <code>str</code> <p>Valid URL, points to actual data file. The file can be on the OpenML server or another dataset repository.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>version_label</code> <code>str</code> <p>Version label provided by user. Can be a date, hash, or some other type of id.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>tag</code> <code>str</code> <p>Tags, describing the algorithms.</p> <code>None</code> <code>visibility</code> <code>str</code> <p>Who can see the dataset. Typical values: 'Everyone','All my friends','Only me'. Can also be any of the user's circles.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <code>md5_checksum</code> <code>str</code> <p>MD5 checksum to check if the dataset is downloaded without corruption.</p> <code>None</code> <code>data_file</code> <code>str</code> <p>Path to where the dataset is located.</p> <code>None</code> <code>features_file</code> <code>dict</code> <p>A dictionary of dataset features, which maps a feature index to a OpenMLDataFeature.</p> <code>None</code> <code>qualities_file</code> <code>dict</code> <p>A dictionary of dataset qualities, which maps a quality name to a quality value.</p> <code>None</code> <code>dataset</code> <code>str | None</code> <p>Serialized arff dataset string.</p> <code>None</code> <code>parquet_url</code> <code>str | None</code> <p>This is the URL to the storage location where the dataset files are hosted. This can be a MinIO bucket URL. If specified, the data will be accessed from this URL when reading the files.</p> <code>None</code> <code>parquet_file</code> <code>str | None</code> <p>Path to the local file.</p> <code>None</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>class OpenMLDataset(OpenMLBase):\n    \"\"\"Dataset object.\n\n    Allows fetching and uploading datasets to OpenML.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    data_format : str\n        Format of the dataset which can be either 'arff' or 'sparse_arff'.\n    cache_format : str\n        Format for caching the dataset which can be either 'feather' or 'pickle'.\n    dataset_id : int, optional\n        Id autogenerated by the server.\n    version : int, optional\n        Version of this dataset. '1' for original version.\n        Auto-incremented by server.\n    creator : str, optional\n        The person who created the dataset.\n    contributor : str, optional\n        People who contributed to the current version of the dataset.\n    collection_date : str, optional\n        The date the data was originally collected, given by the uploader.\n    upload_date : str, optional\n        The date-time when the dataset was uploaded, generated by server.\n    language : str, optional\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str, optional\n        License of the data.\n    url : str, optional\n        Valid URL, points to actual data file.\n        The file can be on the OpenML server or another dataset repository.\n    default_target_attribute : str, optional\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column,\n        if present in the dataset.\n    ignore_attribute : str | list, optional\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    version_label : str, optional\n        Version label provided by user.\n        Can be a date, hash, or some other type of id.\n    citation : str, optional\n        Reference(s) that should be cited when building on this data.\n    tag : str, optional\n        Tags, describing the algorithms.\n    visibility : str, optional\n        Who can see the dataset.\n        Typical values: 'Everyone','All my friends','Only me'.\n        Can also be any of the user's circles.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n    md5_checksum : str, optional\n        MD5 checksum to check if the dataset is downloaded without corruption.\n    data_file : str, optional\n        Path to where the dataset is located.\n    features_file : dict, optional\n        A dictionary of dataset features,\n        which maps a feature index to a OpenMLDataFeature.\n    qualities_file : dict, optional\n        A dictionary of dataset qualities,\n        which maps a quality name to a quality value.\n    dataset: string, optional\n        Serialized arff dataset string.\n    parquet_url: string, optional\n        This is the URL to the storage location where the dataset files are hosted.\n        This can be a MinIO bucket URL. If specified, the data will be accessed\n        from this URL when reading the files.\n    parquet_file: string, optional\n        Path to the local file.\n    \"\"\"\n\n    def __init__(  # noqa: C901, PLR0912, PLR0913, PLR0915\n        self,\n        name: str,\n        description: str | None,\n        data_format: Literal[\"arff\", \"sparse_arff\"] = \"arff\",\n        cache_format: Literal[\"feather\", \"pickle\"] = \"pickle\",\n        dataset_id: int | None = None,\n        version: int | None = None,\n        creator: str | None = None,\n        contributor: str | None = None,\n        collection_date: str | None = None,\n        upload_date: str | None = None,\n        language: str | None = None,\n        licence: str | None = None,\n        url: str | None = None,\n        default_target_attribute: str | None = None,\n        row_id_attribute: str | None = None,\n        ignore_attribute: str | list[str] | None = None,\n        version_label: str | None = None,\n        citation: str | None = None,\n        tag: str | None = None,\n        visibility: str | None = None,\n        original_data_url: str | None = None,\n        paper_url: str | None = None,\n        update_comment: str | None = None,\n        md5_checksum: str | None = None,\n        data_file: str | None = None,\n        features_file: str | None = None,\n        qualities_file: str | None = None,\n        dataset: str | None = None,\n        parquet_url: str | None = None,\n        parquet_file: str | None = None,\n    ):\n        if cache_format not in [\"feather\", \"pickle\"]:\n            raise ValueError(\n                \"cache_format must be one of 'feather' or 'pickle. \"\n                f\"Invalid format specified: {cache_format}\",\n            )\n\n        def find_invalid_characters(string: str, pattern: str) -&gt; str:\n            invalid_chars = set()\n            regex = re.compile(pattern)\n            for char in string:\n                if not regex.match(char):\n                    invalid_chars.add(char)\n            return \",\".join(\n                [f\"'{char}'\" if char != \"'\" else f'\"{char}\"' for char in invalid_chars],\n            )\n\n        if dataset_id is None:\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if description and not re.match(pattern, description):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(description, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in description: {description}\",\n                )\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if citation and not re.match(pattern, citation):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(citation, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in citation: {citation}\",\n                )\n            pattern = \"^[a-zA-Z0-9_\\\\-\\\\.\\\\(\\\\),]+$\"\n            if not re.match(pattern, name):\n                # regex given by server in error message\n                invalid_characters = find_invalid_characters(name, pattern)\n                raise ValueError(f\"Invalid symbols {invalid_characters} in name: {name}\")\n\n        self.ignore_attribute: list[str] | None = None\n        if isinstance(ignore_attribute, str):\n            self.ignore_attribute = [ignore_attribute]\n        elif isinstance(ignore_attribute, list) or ignore_attribute is None:\n            self.ignore_attribute = ignore_attribute\n        else:\n            raise ValueError(\"Wrong data type for ignore_attribute. Should be list.\")\n\n        # TODO add function to check if the name is casual_string128\n        # Attributes received by querying the RESTful API\n        self.dataset_id = int(dataset_id) if dataset_id is not None else None\n        self.name = name\n        self.version = int(version) if version is not None else None\n        self.description = description\n        self.cache_format = cache_format\n        # Has to be called format, otherwise there will be an XML upload error\n        self.format = data_format\n        self.creator = creator\n        self.contributor = contributor\n        self.collection_date = collection_date\n        self.upload_date = upload_date\n        self.language = language\n        self.licence = licence\n        self.url = url\n        self.default_target_attribute = default_target_attribute\n        self.row_id_attribute = row_id_attribute\n\n        self.version_label = version_label\n        self.citation = citation\n        self.tag = tag\n        self.visibility = visibility\n        self.original_data_url = original_data_url\n        self.paper_url = paper_url\n        self.update_comment = update_comment\n        self.md5_checksum = md5_checksum\n        self.data_file = data_file\n        self.parquet_file = parquet_file\n        self._dataset = dataset\n        self._parquet_url = parquet_url\n\n        self._features: dict[int, OpenMLDataFeature] | None = None\n        self._qualities: dict[str, float] | None = None\n        self._no_qualities_found = False\n\n        if features_file is not None:\n            self._features = _read_features(Path(features_file))\n\n        # \"\" was the old default value by `get_dataset` and maybe still used by some\n        if qualities_file == \"\":\n            # TODO(0.15): to switch to \"qualities_file is not None\" below and remove warning\n            warnings.warn(\n                \"Starting from Version 0.15 `qualities_file` must be None and not an empty string \"\n                \"to avoid reading the qualities from file. Set `qualities_file` to None to avoid \"\n                \"this warning.\",\n                FutureWarning,\n                stacklevel=2,\n            )\n            qualities_file = None\n\n        if qualities_file is not None:\n            self._qualities = _read_qualities(Path(qualities_file))\n\n        if data_file is not None:\n            data_pickle, data_feather, feather_attribute = self._compressed_cache_file_paths(\n                Path(data_file)\n            )\n            self.data_pickle_file = data_pickle if Path(data_pickle).exists() else None\n            self.data_feather_file = data_feather if Path(data_feather).exists() else None\n            self.feather_attribute_file = feather_attribute if Path(feather_attribute) else None\n        else:\n            self.data_pickle_file = None\n            self.data_feather_file = None\n            self.feather_attribute_file = None\n\n    @property\n    def features(self) -&gt; dict[int, OpenMLDataFeature]:\n        \"\"\"Get the features of this dataset.\"\"\"\n        if self._features is None:\n            # TODO(eddiebergman): These should return a value so we can set it to be not None\n            self._load_features()\n\n        assert self._features is not None\n        return self._features\n\n    @property\n    def qualities(self) -&gt; dict[str, float] | None:\n        \"\"\"Get the qualities of this dataset.\"\"\"\n        # TODO(eddiebergman): Better docstring, I don't know what qualities means\n\n        # We have to check `_no_qualities_found` as there might not be qualities for a dataset\n        if self._qualities is None and (not self._no_qualities_found):\n            self._load_qualities()\n\n        return self._qualities\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Get the dataset numeric id.\"\"\"\n        return self.dataset_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Obtain number of features in accordance with lazy loading.\n        n_features: int | None = None\n        if self._qualities is not None and self._qualities[\"NumberOfFeatures\"] is not None:\n            n_features = int(self._qualities[\"NumberOfFeatures\"])\n        elif self._features is not None:\n            n_features = len(self._features)\n\n        fields: dict[str, int | str | None] = {\n            \"Name\": self.name,\n            \"Version\": self.version,\n            \"Format\": self.format,\n            \"Licence\": self.licence,\n            \"Download URL\": self.url,\n            \"Data file\": str(self.data_file) if self.data_file is not None else None,\n            \"Pickle file\": (\n                str(self.data_pickle_file) if self.data_pickle_file is not None else None\n            ),\n            \"# of features\": n_features,\n        }\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.dataset_id is not None:\n            fields[\"OpenML URL\"] = self.openml_url\n        if self._qualities is not None and self._qualities[\"NumberOfInstances\"] is not None:\n            fields[\"# of instances\"] = int(self._qualities[\"NumberOfInstances\"])\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Name\",\n            \"Version\",\n            \"Format\",\n            \"Upload Date\",\n            \"Licence\",\n            \"Download URL\",\n            \"OpenML URL\",\n            \"Data File\",\n            \"Pickle File\",\n            \"# of features\",\n            \"# of instances\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if not isinstance(other, OpenMLDataset):\n            return False\n\n        server_fields = {\n            \"dataset_id\",\n            \"version\",\n            \"upload_date\",\n            \"url\",\n            \"dataset\",\n            \"data_file\",\n        }\n\n        # check that common keys and values are identical\n        self_keys = set(self.__dict__.keys()) - server_fields\n        other_keys = set(other.__dict__.keys()) - server_fields\n        return self_keys == other_keys and all(\n            self.__dict__[key] == other.__dict__[key] for key in self_keys\n        )\n\n    def _download_data(self) -&gt; None:\n        \"\"\"Download ARFF data file to standard cache directory. Set `self.data_file`.\"\"\"\n        # import required here to avoid circular import.\n        from .functions import _get_dataset_arff, _get_dataset_parquet\n\n        self.data_file = str(_get_dataset_arff(self))\n        if self._parquet_url is not None:\n            self.parquet_file = str(_get_dataset_parquet(self))\n\n    def _get_arff(self, format: str) -&gt; dict:  # noqa: A002\n        \"\"\"Read ARFF file and return decoded arff.\n\n        Reads the file referenced in self.data_file.\n\n        Parameters\n        ----------\n        format : str\n            Format of the ARFF file.\n            Must be one of 'arff' or 'sparse_arff' or a string that will be either of those\n            when converted to lower case.\n\n\n\n        Returns\n        -------\n        dict\n            Decoded arff.\n\n        \"\"\"\n        # TODO: add a partial read method which only returns the attribute\n        # headers of the corresponding .arff file!\n        import struct\n\n        filename = self.data_file\n        assert filename is not None\n        filepath = Path(filename)\n\n        bits = 8 * struct.calcsize(\"P\")\n\n        # Files can be considered too large on a 32-bit system,\n        # if it exceeds 120mb (slightly more than covtype dataset size)\n        # This number is somewhat arbitrary.\n        if bits != 64:\n            MB_120 = 120_000_000\n            file_size = filepath.stat().st_size\n            if file_size &gt; MB_120:\n                raise NotImplementedError(\n                    f\"File {filename} too big for {file_size}-bit system ({bits} bytes).\",\n                )\n\n        if format.lower() == \"arff\":\n            return_type = arff.DENSE\n        elif format.lower() == \"sparse_arff\":\n            return_type = arff.COO\n        else:\n            raise ValueError(f\"Unknown data format {format}\")\n\n        def decode_arff(fh: Any) -&gt; dict:\n            decoder = arff.ArffDecoder()\n            return decoder.decode(fh, encode_nominal=True, return_type=return_type)  # type: ignore\n\n        if filepath.suffix.endswith(\".gz\"):\n            with gzip.open(filename) as zipfile:\n                return decode_arff(zipfile)\n        else:\n            with filepath.open(encoding=\"utf8\") as fh:\n                return decode_arff(fh)\n\n    def _parse_data_from_arff(  # noqa: C901, PLR0912, PLR0915\n        self,\n        arff_file_path: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Parse all required data from arff file.\n\n        Parameters\n        ----------\n        arff_file_path : str\n            Path to the file on disk.\n\n        Returns\n        -------\n        Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]\n            DataFrame or csr_matrix: dataset\n            List[bool]: List indicating which columns contain categorical variables.\n            List[str]: List of column names.\n        \"\"\"\n        try:\n            data = self._get_arff(self.format)\n        except OSError as e:\n            logger.critical(\n                f\"Please check that the data file {arff_file_path} is \" \"there and can be read.\",\n            )\n            raise e\n\n        ARFF_DTYPES_TO_PD_DTYPE = {\n            \"INTEGER\": \"integer\",\n            \"REAL\": \"floating\",\n            \"NUMERIC\": \"floating\",\n            \"STRING\": \"string\",\n        }\n        attribute_dtype = {}\n        attribute_names = []\n        categories_names = {}\n        categorical = []\n        for name, type_ in data[\"attributes\"]:\n            # if the feature is nominal and a sparse matrix is\n            # requested, the categories need to be numeric\n            if isinstance(type_, list) and self.format.lower() == \"sparse_arff\":\n                try:\n                    # checks if the strings which should be the class labels\n                    # can be encoded into integers\n                    pd.factorize(type_)[0]\n                except ValueError as e:\n                    raise ValueError(\n                        \"Categorical data needs to be numeric when using sparse ARFF.\"\n                    ) from e\n\n            # string can only be supported with pandas DataFrame\n            elif type_ == \"STRING\" and self.format.lower() == \"sparse_arff\":\n                raise ValueError(\"Dataset containing strings is not supported with sparse ARFF.\")\n\n            # infer the dtype from the ARFF header\n            if isinstance(type_, list):\n                categorical.append(True)\n                categories_names[name] = type_\n                if len(type_) == 2:\n                    type_norm = [cat.lower().capitalize() for cat in type_]\n                    if {\"True\", \"False\"} == set(type_norm):\n                        categories_names[name] = [cat == \"True\" for cat in type_norm]\n                        attribute_dtype[name] = \"boolean\"\n                    else:\n                        attribute_dtype[name] = \"categorical\"\n                else:\n                    attribute_dtype[name] = \"categorical\"\n            else:\n                categorical.append(False)\n                attribute_dtype[name] = ARFF_DTYPES_TO_PD_DTYPE[type_]\n            attribute_names.append(name)\n\n        if self.format.lower() == \"sparse_arff\":\n            X = data[\"data\"]\n            X_shape = (max(X[1]) + 1, max(X[2]) + 1)\n            X = scipy.sparse.coo_matrix((X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32)\n            X = X.tocsr()\n        elif self.format.lower() == \"arff\":\n            X = pd.DataFrame(data[\"data\"], columns=attribute_names)\n\n            col = []\n            for column_name in X.columns:\n                if attribute_dtype[column_name] in (\"categorical\", \"boolean\"):\n                    categories = self._unpack_categories(\n                        X[column_name],  # type: ignore\n                        categories_names[column_name],\n                    )\n                    col.append(categories)\n                elif attribute_dtype[column_name] in (\"floating\", \"integer\"):\n                    X_col = X[column_name]\n                    if X_col.min() &gt;= 0 and X_col.max() &lt;= 255:\n                        try:\n                            X_col_uint = X_col.astype(\"uint8\")\n                            if (X_col == X_col_uint).all():\n                                col.append(X_col_uint)\n                                continue\n                        except ValueError:\n                            pass\n                    col.append(X[column_name])\n                else:\n                    col.append(X[column_name])\n            X = pd.concat(col, axis=1)\n        else:\n            raise ValueError(f\"Dataset format '{self.format}' is not a valid format.\")\n\n        return X, categorical, attribute_names  # type: ignore\n\n    def _compressed_cache_file_paths(self, data_file: Path) -&gt; tuple[Path, Path, Path]:\n        data_pickle_file = data_file.with_suffix(\".pkl.py3\")\n        data_feather_file = data_file.with_suffix(\".feather\")\n        feather_attribute_file = data_file.with_suffix(\".feather.attributes.pkl.py3\")\n        return data_pickle_file, data_feather_file, feather_attribute_file\n\n    def _cache_compressed_file_from_file(\n        self,\n        data_file: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Store data from the local file in compressed format.\n\n        If a local parquet file is present it will be used instead of the arff file.\n        Sets cache_format to 'pickle' if data is sparse.\n        \"\"\"\n        (\n            data_pickle_file,\n            data_feather_file,\n            feather_attribute_file,\n        ) = self._compressed_cache_file_paths(data_file)\n\n        if data_file.suffix == \".arff\":\n            data, categorical, attribute_names = self._parse_data_from_arff(data_file)\n        elif data_file.suffix == \".pq\":\n            try:\n                data = pd.read_parquet(data_file)\n            except Exception as e:  # noqa: BLE001\n                raise Exception(f\"File: {data_file}\") from e\n\n            categorical = [data[c].dtype.name == \"category\" for c in data.columns]\n            attribute_names = list(data.columns)\n        else:\n            raise ValueError(f\"Unknown file type for file '{data_file}'.\")\n\n        # Feather format does not work for sparse datasets, so we use pickle for sparse datasets\n        if scipy.sparse.issparse(data):\n            self.cache_format = \"pickle\"\n\n        logger.info(f\"{self.cache_format} write {self.name}\")\n        if self.cache_format == \"feather\":\n            assert isinstance(data, pd.DataFrame)\n\n            data.to_feather(data_feather_file)\n            with open(feather_attribute_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_feather_file = data_feather_file\n            self.feather_attribute_file = feather_attribute_file\n\n        else:\n            with open(data_pickle_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_pickle_file = data_pickle_file\n\n        data_file = data_pickle_file if self.cache_format == \"pickle\" else data_feather_file\n        logger.debug(f\"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}\")\n\n        return data, categorical, attribute_names\n\n    def _load_data(self) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:  # noqa: PLR0912, C901\n        \"\"\"Load data from compressed format or arff. Download data if not present on disk.\"\"\"\n        need_to_create_pickle = self.cache_format == \"pickle\" and self.data_pickle_file is None\n        need_to_create_feather = self.cache_format == \"feather\" and self.data_feather_file is None\n\n        if need_to_create_pickle or need_to_create_feather:\n            if self.data_file is None:\n                self._download_data()\n\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n\n        # helper variable to help identify where errors occur\n        fpath = self.data_feather_file if self.cache_format == \"feather\" else self.data_pickle_file\n        logger.info(f\"{self.cache_format} load data {self.name}\")\n        try:\n            if self.cache_format == \"feather\":\n                assert self.data_feather_file is not None\n                assert self.feather_attribute_file is not None\n\n                data = pd.read_feather(self.data_feather_file)\n                fpath = self.feather_attribute_file\n                with open(self.feather_attribute_file, \"rb\") as fh:  # noqa: PTH123\n                    categorical, attribute_names = pickle.load(fh)  # noqa: S301\n            else:\n                assert self.data_pickle_file is not None\n                with open(self.data_pickle_file, \"rb\") as fh:  # noqa: PTH123\n                    data, categorical, attribute_names = pickle.load(fh)  # noqa: S301\n        except FileNotFoundError as e:\n            raise ValueError(\n                f\"Cannot find file for dataset {self.name} at location '{fpath}'.\"\n            ) from e\n        except (EOFError, ModuleNotFoundError, ValueError, AttributeError) as e:\n            error_message = getattr(e, \"message\", e.args[0])\n            hint = \"\"\n\n            if isinstance(e, EOFError):\n                readable_error = \"Detected a corrupt cache file\"\n            elif isinstance(e, (ModuleNotFoundError, AttributeError)):\n                readable_error = \"Detected likely dependency issues\"\n                hint = (\n                    \"This can happen if the cache was constructed with a different pandas version \"\n                    \"than the one that is used to load the data. See also \"\n                )\n                if isinstance(e, ModuleNotFoundError):\n                    hint += \"https://github.com/openml/openml-python/issues/918. \"\n                elif isinstance(e, AttributeError):\n                    hint += \"https://github.com/openml/openml-python/pull/1121. \"\n\n            elif isinstance(e, ValueError) and \"unsupported pickle protocol\" in e.args[0]:\n                readable_error = \"Encountered unsupported pickle protocol\"\n            else:\n                raise e\n\n            logger.warning(\n                f\"{readable_error} when loading dataset {self.id} from '{fpath}'. \"\n                f\"{hint}\"\n                f\"Error message was: {error_message}. \"\n                \"We will continue loading data from the arff-file, \"\n                \"but this will be much slower for big datasets. \"\n                \"Please manually delete the cache file if you want OpenML-Python \"\n                \"to attempt to reconstruct it.\",\n            )\n            assert self.data_file is not None\n            data, categorical, attribute_names = self._parse_data_from_arff(Path(self.data_file))\n\n        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)\n        if self.cache_format == \"pickle\" and not data_up_to_date:\n            logger.info(\"Updating outdated pickle file.\")\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n        return data, categorical, attribute_names\n\n    # TODO(eddiebergman): Can type this better with overload\n    # TODO(eddiebergman): Could also techinically use scipy.sparse.sparray\n    @staticmethod\n    def _convert_array_format(\n        data: pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix,\n        array_format: Literal[\"array\", \"dataframe\"],\n        attribute_names: list | None = None,\n    ) -&gt; pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix:\n        \"\"\"Convert a dataset to a given array format.\n\n        Converts to numpy array if data is non-sparse.\n        Converts to a sparse dataframe if data is sparse.\n\n        Parameters\n        ----------\n        array_format : str {'array', 'dataframe'}\n            Desired data type of the output\n            - If array_format='array'\n                If data is non-sparse\n                    Converts to numpy-array\n                    Enforces numeric encoding of categorical columns\n                    Missing values are represented as NaN in the numpy-array\n                else returns data as is\n            - If array_format='dataframe'\n                If data is sparse\n                    Works only on sparse data\n                    Converts sparse data to sparse dataframe\n                else returns data as is\n\n        \"\"\"\n        if array_format == \"array\" and not isinstance(data, scipy.sparse.spmatrix):\n            # We encode the categories such that they are integer to be able\n            # to make a conversion to numeric for backward compatibility\n            def _encode_if_category(column: pd.Series | np.ndarray) -&gt; pd.Series | np.ndarray:\n                if column.dtype.name == \"category\":\n                    column = column.cat.codes.astype(np.float32)\n                    mask_nan = column == -1\n                    column[mask_nan] = np.nan\n                return column\n\n            if isinstance(data, pd.DataFrame):\n                columns = {\n                    column_name: _encode_if_category(data.loc[:, column_name])\n                    for column_name in data.columns\n                }\n                data = pd.DataFrame(columns)\n            else:\n                data = _encode_if_category(data)\n\n            try:\n                # TODO(eddiebergman): float32?\n                return_array = np.asarray(data, dtype=np.float32)\n            except ValueError as e:\n                raise PyOpenMLError(\n                    \"PyOpenML cannot handle string when returning numpy\"\n                    ' arrays. Use dataset_format=\"dataframe\".',\n                ) from e\n\n            return return_array\n\n        if array_format == \"dataframe\":\n            if scipy.sparse.issparse(data):\n                data = pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)\n        else:\n            data_type = \"sparse-data\" if scipy.sparse.issparse(data) else \"non-sparse data\"\n            logger.warning(\n                f\"Cannot convert {data_type} ({type(data)}) to '{array_format}'.\"\n                \" Returning input data.\",\n            )\n        return data\n\n    @staticmethod\n    def _unpack_categories(series: pd.Series, categories: list) -&gt; pd.Series:\n        # nan-likes can not be explicitly specified as a category\n        def valid_category(cat: Any) -&gt; bool:\n            return isinstance(cat, str) or (cat is not None and not np.isnan(cat))\n\n        filtered_categories = [c for c in categories if valid_category(c)]\n        col = []\n        for x in series:\n            try:\n                col.append(categories[int(x)])\n            except (TypeError, ValueError):\n                col.append(np.nan)\n\n        # We require two lines to create a series of categories as detailed here:\n        # https://pandas.pydata.org/pandas-docs/version/0.24/user_guide/categorical.html#series-creation\n        raw_cat = pd.Categorical(col, ordered=True, categories=filtered_categories)\n        return pd.Series(raw_cat, index=series.index, name=series.name)\n\n    def get_data(  # noqa: C901, PLR0912, PLR0915\n        self,\n        target: list[str] | str | None = None,\n        include_row_id: bool = False,  # noqa: FBT001, FBT002\n        include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n        np.ndarray | pd.DataFrame | None,\n        list[bool],\n        list[str],\n    ]:\n        \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n        Parameters\n        ----------\n        target : string, List[str] or None (default=None)\n            Name of target column to separate from the data.\n            Splitting multiple columns is currently not supported.\n        include_row_id : boolean (default=False)\n            Whether to include row ids in the returned dataset.\n        include_ignore_attribute : boolean (default=False)\n            Whether to include columns that are marked as \"ignore\"\n            on the server in the dataset.\n        dataset_format : string (default='dataframe')\n            The format of returned dataset.\n            If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n            matrix. Support for ``array`` will be removed in 0.15.\n            If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n        Returns\n        -------\n        X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n            Dataset\n        y : ndarray or pd.Series, shape (n_samples, ) or None\n            Target column\n        categorical_indicator : boolean ndarray\n            Mask that indicate categorical features.\n        attribute_names : List[str]\n            List of attribute names.\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        data, categorical, attribute_names = self._load_data()\n\n        to_exclude = []\n        if not include_row_id and self.row_id_attribute is not None:\n            if isinstance(self.row_id_attribute, str):\n                to_exclude.append(self.row_id_attribute)\n            elif isinstance(self.row_id_attribute, Iterable):\n                to_exclude.extend(self.row_id_attribute)\n\n        if not include_ignore_attribute and self.ignore_attribute is not None:\n            if isinstance(self.ignore_attribute, str):\n                to_exclude.append(self.ignore_attribute)\n            elif isinstance(self.ignore_attribute, Iterable):\n                to_exclude.extend(self.ignore_attribute)\n\n        if len(to_exclude) &gt; 0:\n            logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n            keep = np.array([column not in to_exclude for column in attribute_names])\n            data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n            categorical = [cat for cat, k in zip(categorical, keep) if k]\n            attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n        if target is None:\n            data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n            targets = None\n        else:\n            if isinstance(target, str):\n                target = target.split(\",\") if \",\" in target else [target]\n            targets = np.array([column in target for column in attribute_names])\n            target_names = [column for column in attribute_names if column in target]\n            if np.sum(targets) &gt; 1:\n                raise NotImplementedError(\n                    \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n                )\n            target_categorical = [\n                cat for cat, column in zip(categorical, attribute_names) if column in target\n            ]\n            target_dtype = int if target_categorical[0] else float\n\n            if isinstance(data, pd.DataFrame):\n                x = data.iloc[:, ~targets]\n                y = data.iloc[:, targets]\n            else:\n                x = data[:, ~targets]\n                y = data[:, targets].astype(target_dtype)  # type: ignore\n\n            categorical = [cat for cat, t in zip(categorical, targets) if not t]\n            attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n            x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n            if dataset_format == \"array\" and scipy.sparse.issparse(y):\n                # scikit-learn requires dense representation of targets\n                y = np.asarray(y.todense()).astype(target_dtype)\n                # dense representation of single column sparse arrays become a 2-d array\n                # need to flatten it to a 1-d array for _convert_array_format()\n                y = y.squeeze()\n            y = self._convert_array_format(y, dataset_format, target_names)\n            y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n            if len(y.shape) &gt; 1 and y.shape[1] == 1:\n                # single column targets should be 1-d for both `array` and `dataframe` formats\n                assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n                y = y.squeeze()\n            data, targets = x, y\n\n        return data, targets, categorical, attribute_names  # type: ignore\n\n    def _load_features(self) -&gt; None:\n        \"\"\"Load the features metadata from the server and store it in the dataset object.\"\"\"\n        # Delayed Import to avoid circular imports or having to import all of dataset.functions to\n        # import OpenMLDataset.\n        from openml.datasets.functions import _get_dataset_features_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        features_file = _get_dataset_features_file(None, self.dataset_id)\n        self._features = _read_features(features_file)\n\n    def _load_qualities(self) -&gt; None:\n        \"\"\"Load qualities information from the server and store it in the dataset object.\"\"\"\n        # same reason as above for _load_features\n        from openml.datasets.functions import _get_dataset_qualities_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        qualities_file = _get_dataset_qualities_file(None, self.dataset_id)\n\n        if qualities_file is None:\n            self._no_qualities_found = True\n        else:\n            self._qualities = _read_qualities(qualities_file)\n\n    def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n        \"\"\"Reads the datasets arff to determine the class-labels.\n\n        If the task has no class labels (for example a regression problem)\n        it returns None. Necessary because the data returned by get_data\n        only contains the indices of the classes, while OpenML needs the real\n        classname when uploading the results of a run.\n\n        Parameters\n        ----------\n        target_name : str\n            Name of the target attribute\n\n        Returns\n        -------\n        list\n        \"\"\"\n        for feature in self.features.values():\n            if feature.name == target_name:\n                if feature.data_type == \"nominal\":\n                    return feature.nominal_values\n\n                if feature.data_type == \"string\":\n                    # Rel.: #1311\n                    # The target is invalid for a classification task if the feature type is string\n                    # and not nominal. For such miss-configured tasks, we silently fix it here as\n                    # we can safely interpreter string as nominal.\n                    df, *_ = self.get_data()\n                    return list(df[feature.name].unique())\n\n        return None\n\n    def get_features_by_type(  # noqa: C901\n        self,\n        data_type: str,\n        exclude: list[str] | None = None,\n        exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n        exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; list[int]:\n        \"\"\"\n        Return indices of features of a given type, e.g. all nominal features.\n        Optional parameters to exclude various features by index or ontology.\n\n        Parameters\n        ----------\n        data_type : str\n            The data type to return (e.g., nominal, numeric, date, string)\n        exclude : list(int)\n            List of columns to exclude from the return value\n        exclude_ignore_attribute : bool\n            Whether to exclude the defined ignore attributes (and adapt the\n            return values as if these indices are not present)\n        exclude_row_id_attribute : bool\n            Whether to exclude the defined row id attributes (and adapt the\n            return values as if these indices are not present)\n\n        Returns\n        -------\n        result : list\n            a list of indices that have the specified data type\n        \"\"\"\n        if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n            raise TypeError(\"Illegal feature type requested\")\n        if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n            raise TypeError(\"ignore_attribute should be a list\")\n        if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n            raise TypeError(\"row id attribute should be a str\")\n        if exclude is not None and not isinstance(exclude, list):\n            raise TypeError(\"Exclude should be a list\")\n            # assert all(isinstance(elem, str) for elem in exclude),\n            #            \"Exclude should be a list of strings\"\n        to_exclude = []\n        if exclude is not None:\n            to_exclude.extend(exclude)\n        if exclude_ignore_attribute and self.ignore_attribute is not None:\n            to_exclude.extend(self.ignore_attribute)\n        if exclude_row_id_attribute and self.row_id_attribute is not None:\n            to_exclude.append(self.row_id_attribute)\n\n        result = []\n        offset = 0\n        # this function assumes that everything in to_exclude will\n        # be 'excluded' from the dataset (hence the offset)\n        for idx in self.features:\n            name = self.features[idx].name\n            if name in to_exclude:\n                offset += 1\n            elif self.features[idx].data_type == data_type:\n                result.append(idx - offset)\n        return result\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Adds the 'dataset' to file elements.\"\"\"\n        file_elements: dict = {}\n        path = None if self.data_file is None else Path(self.data_file).absolute()\n\n        if self._dataset is not None:\n            file_elements[\"dataset\"] = self._dataset\n        elif path is not None and path.exists():\n            with path.open(\"rb\") as fp:\n                file_elements[\"dataset\"] = fp.read()\n\n            try:\n                dataset_utf8 = str(file_elements[\"dataset\"], encoding=\"utf8\")\n                arff.ArffDecoder().decode(dataset_utf8, encode_nominal=True)\n            except arff.ArffException as e:\n                raise ValueError(\"The file you have provided is not a valid arff file.\") from e\n\n        elif self.url is None:\n            raise ValueError(\"No valid url/path to the data file was given.\")\n        return file_elements\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.dataset_id = int(xml_response[\"oml:upload_data_set\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        props = [\n            \"id\",\n            \"name\",\n            \"version\",\n            \"description\",\n            \"format\",\n            \"creator\",\n            \"contributor\",\n            \"collection_date\",\n            \"upload_date\",\n            \"language\",\n            \"licence\",\n            \"url\",\n            \"default_target_attribute\",\n            \"row_id_attribute\",\n            \"ignore_attribute\",\n            \"version_label\",\n            \"citation\",\n            \"tag\",\n            \"visibility\",\n            \"original_data_url\",\n            \"paper_url\",\n            \"update_comment\",\n            \"md5_checksum\",\n        ]\n\n        prop_values = {}\n        for prop in props:\n            content = getattr(self, prop, None)\n            if content is not None:\n                prop_values[\"oml:\" + prop] = content\n\n        return {\n            \"oml:data_set_description\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **prop_values,\n            }\n        }\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset.features","title":"<code>features: dict[int, OpenMLDataFeature]</code>  <code>property</code>","text":"<p>Get the features of this dataset.</p>"},{"location":"reference/#openml.OpenMLDataset.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Get the dataset numeric id.</p>"},{"location":"reference/#openml.OpenMLDataset.qualities","title":"<code>qualities: dict[str, float] | None</code>  <code>property</code>","text":"<p>Get the qualities of this dataset.</p>"},{"location":"reference/#openml.OpenMLDataset.get_data","title":"<code>get_data(target=None, include_row_id=False, include_ignore_attribute=False, dataset_format='dataframe')</code>","text":"<p>Returns dataset content as dataframes or sparse matrices.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>(string, List[str] or None(default=None))</code> <p>Name of target column to separate from the data. Splitting multiple columns is currently not supported.</p> <code>None</code> <code>include_row_id</code> <code>boolean(default=False)</code> <p>Whether to include row ids in the returned dataset.</p> <code>False</code> <code>include_ignore_attribute</code> <code>boolean(default=False)</code> <p>Whether to include columns that are marked as \"ignore\" on the server in the dataset.</p> <code>False</code> <code>dataset_format</code> <code>string(default='dataframe')</code> <p>The format of returned dataset. If <code>array</code>, the returned dataset will be a NumPy array or a SciPy sparse matrix. Support for <code>array</code> will be removed in 0.15. If <code>dataframe</code>, the returned dataset will be a Pandas DataFrame.</p> <code>'dataframe'</code> <p>Returns:</p> Name Type Description <code>X</code> <code>ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)</code> <p>Dataset</p> <code>y</code> <code>(ndarray or Series, shape(n_samples) or None)</code> <p>Target column</p> <code>categorical_indicator</code> <code>boolean ndarray</code> <p>Mask that indicate categorical features.</p> <code>attribute_names</code> <code>List[str]</code> <p>List of attribute names.</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_data(  # noqa: C901, PLR0912, PLR0915\n    self,\n    target: list[str] | str | None = None,\n    include_row_id: bool = False,  # noqa: FBT001, FBT002\n    include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n    np.ndarray | pd.DataFrame | None,\n    list[bool],\n    list[str],\n]:\n    \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n    Parameters\n    ----------\n    target : string, List[str] or None (default=None)\n        Name of target column to separate from the data.\n        Splitting multiple columns is currently not supported.\n    include_row_id : boolean (default=False)\n        Whether to include row ids in the returned dataset.\n    include_ignore_attribute : boolean (default=False)\n        Whether to include columns that are marked as \"ignore\"\n        on the server in the dataset.\n    dataset_format : string (default='dataframe')\n        The format of returned dataset.\n        If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n        matrix. Support for ``array`` will be removed in 0.15.\n        If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n    Returns\n    -------\n    X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n        Dataset\n    y : ndarray or pd.Series, shape (n_samples, ) or None\n        Target column\n    categorical_indicator : boolean ndarray\n        Mask that indicate categorical features.\n    attribute_names : List[str]\n        List of attribute names.\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    data, categorical, attribute_names = self._load_data()\n\n    to_exclude = []\n    if not include_row_id and self.row_id_attribute is not None:\n        if isinstance(self.row_id_attribute, str):\n            to_exclude.append(self.row_id_attribute)\n        elif isinstance(self.row_id_attribute, Iterable):\n            to_exclude.extend(self.row_id_attribute)\n\n    if not include_ignore_attribute and self.ignore_attribute is not None:\n        if isinstance(self.ignore_attribute, str):\n            to_exclude.append(self.ignore_attribute)\n        elif isinstance(self.ignore_attribute, Iterable):\n            to_exclude.extend(self.ignore_attribute)\n\n    if len(to_exclude) &gt; 0:\n        logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n        keep = np.array([column not in to_exclude for column in attribute_names])\n        data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n        categorical = [cat for cat, k in zip(categorical, keep) if k]\n        attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n    if target is None:\n        data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n        targets = None\n    else:\n        if isinstance(target, str):\n            target = target.split(\",\") if \",\" in target else [target]\n        targets = np.array([column in target for column in attribute_names])\n        target_names = [column for column in attribute_names if column in target]\n        if np.sum(targets) &gt; 1:\n            raise NotImplementedError(\n                \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n            )\n        target_categorical = [\n            cat for cat, column in zip(categorical, attribute_names) if column in target\n        ]\n        target_dtype = int if target_categorical[0] else float\n\n        if isinstance(data, pd.DataFrame):\n            x = data.iloc[:, ~targets]\n            y = data.iloc[:, targets]\n        else:\n            x = data[:, ~targets]\n            y = data[:, targets].astype(target_dtype)  # type: ignore\n\n        categorical = [cat for cat, t in zip(categorical, targets) if not t]\n        attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n        x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n        if dataset_format == \"array\" and scipy.sparse.issparse(y):\n            # scikit-learn requires dense representation of targets\n            y = np.asarray(y.todense()).astype(target_dtype)\n            # dense representation of single column sparse arrays become a 2-d array\n            # need to flatten it to a 1-d array for _convert_array_format()\n            y = y.squeeze()\n        y = self._convert_array_format(y, dataset_format, target_names)\n        y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n        if len(y.shape) &gt; 1 and y.shape[1] == 1:\n            # single column targets should be 1-d for both `array` and `dataframe` formats\n            assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n            y = y.squeeze()\n        data, targets = x, y\n\n    return data, targets, categorical, attribute_names  # type: ignore\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset.get_features_by_type","title":"<code>get_features_by_type(data_type, exclude=None, exclude_ignore_attribute=True, exclude_row_id_attribute=True)</code>","text":"<p>Return indices of features of a given type, e.g. all nominal features. Optional parameters to exclude various features by index or ontology.</p> <p>Parameters:</p> Name Type Description Default <code>data_type</code> <code>str</code> <p>The data type to return (e.g., nominal, numeric, date, string)</p> required <code>exclude</code> <code>list(int)</code> <p>List of columns to exclude from the return value</p> <code>None</code> <code>exclude_ignore_attribute</code> <code>bool</code> <p>Whether to exclude the defined ignore attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <code>exclude_row_id_attribute</code> <code>bool</code> <p>Whether to exclude the defined row id attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <p>Returns:</p> Name Type Description <code>result</code> <code>list</code> <p>a list of indices that have the specified data type</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_features_by_type(  # noqa: C901\n    self,\n    data_type: str,\n    exclude: list[str] | None = None,\n    exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n    exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n) -&gt; list[int]:\n    \"\"\"\n    Return indices of features of a given type, e.g. all nominal features.\n    Optional parameters to exclude various features by index or ontology.\n\n    Parameters\n    ----------\n    data_type : str\n        The data type to return (e.g., nominal, numeric, date, string)\n    exclude : list(int)\n        List of columns to exclude from the return value\n    exclude_ignore_attribute : bool\n        Whether to exclude the defined ignore attributes (and adapt the\n        return values as if these indices are not present)\n    exclude_row_id_attribute : bool\n        Whether to exclude the defined row id attributes (and adapt the\n        return values as if these indices are not present)\n\n    Returns\n    -------\n    result : list\n        a list of indices that have the specified data type\n    \"\"\"\n    if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n        raise TypeError(\"Illegal feature type requested\")\n    if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n        raise TypeError(\"ignore_attribute should be a list\")\n    if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n        raise TypeError(\"row id attribute should be a str\")\n    if exclude is not None and not isinstance(exclude, list):\n        raise TypeError(\"Exclude should be a list\")\n        # assert all(isinstance(elem, str) for elem in exclude),\n        #            \"Exclude should be a list of strings\"\n    to_exclude = []\n    if exclude is not None:\n        to_exclude.extend(exclude)\n    if exclude_ignore_attribute and self.ignore_attribute is not None:\n        to_exclude.extend(self.ignore_attribute)\n    if exclude_row_id_attribute and self.row_id_attribute is not None:\n        to_exclude.append(self.row_id_attribute)\n\n    result = []\n    offset = 0\n    # this function assumes that everything in to_exclude will\n    # be 'excluded' from the dataset (hence the offset)\n    for idx in self.features:\n        name = self.features[idx].name\n        if name in to_exclude:\n            offset += 1\n        elif self.features[idx].data_type == data_type:\n            result.append(idx - offset)\n    return result\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset.retrieve_class_labels","title":"<code>retrieve_class_labels(target_name='class')</code>","text":"<p>Reads the datasets arff to determine the class-labels.</p> <p>If the task has no class labels (for example a regression problem) it returns None. Necessary because the data returned by get_data only contains the indices of the classes, while OpenML needs the real classname when uploading the results of a run.</p> <p>Parameters:</p> Name Type Description Default <code>target_name</code> <code>str</code> <p>Name of the target attribute</p> <code>'class'</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n    \"\"\"Reads the datasets arff to determine the class-labels.\n\n    If the task has no class labels (for example a regression problem)\n    it returns None. Necessary because the data returned by get_data\n    only contains the indices of the classes, while OpenML needs the real\n    classname when uploading the results of a run.\n\n    Parameters\n    ----------\n    target_name : str\n        Name of the target attribute\n\n    Returns\n    -------\n    list\n    \"\"\"\n    for feature in self.features.values():\n        if feature.name == target_name:\n            if feature.data_type == \"nominal\":\n                return feature.nominal_values\n\n            if feature.data_type == \"string\":\n                # Rel.: #1311\n                # The target is invalid for a classification task if the feature type is string\n                # and not nominal. For such miss-configured tasks, we silently fix it here as\n                # we can safely interpreter string as nominal.\n                df, *_ = self.get_data()\n                return list(df[feature.name].unique())\n\n    return None\n</code></pre>"},{"location":"reference/#openml.OpenMLEvaluation","title":"<code>OpenMLEvaluation</code>","text":"<p>Contains all meta-information about a run / evaluation combination, according to the evaluation/list function</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Refers to the run.</p> required <code>task_id</code> <code>int</code> <p>Refers to the task.</p> required <code>setup_id</code> <code>int</code> <p>Refers to the setup.</p> required <code>flow_id</code> <code>int</code> <p>Refers to the flow.</p> required <code>flow_name</code> <code>str</code> <p>Name of the referred flow.</p> required <code>data_id</code> <code>int</code> <p>Refers to the dataset.</p> required <code>data_name</code> <code>str</code> <p>The name of the dataset.</p> required <code>function</code> <code>str</code> <p>The evaluation metric of this item (e.g., accuracy).</p> required <code>upload_time</code> <code>str</code> <p>The time of evaluation.</p> required <code>uploader</code> <code>int</code> <p>Uploader ID (user ID)</p> required <code>upload_name</code> <code>str</code> <p>Name of the uploader of this evaluation</p> required <code>value</code> <code>float</code> <p>The value (score) of this evaluation.</p> required <code>values</code> <code>List[float]</code> <p>The values (scores) per repeat and fold (if requested)</p> required <code>array_data</code> <code>str</code> <p>list of information per class. (e.g., in case of precision, auroc, recall)</p> <code>None</code> Source code in <code>openml/evaluations/evaluation.py</code> <pre><code>class OpenMLEvaluation:\n    \"\"\"\n    Contains all meta-information about a run / evaluation combination,\n    according to the evaluation/list function\n\n    Parameters\n    ----------\n    run_id : int\n        Refers to the run.\n    task_id : int\n        Refers to the task.\n    setup_id : int\n        Refers to the setup.\n    flow_id : int\n        Refers to the flow.\n    flow_name : str\n        Name of the referred flow.\n    data_id : int\n        Refers to the dataset.\n    data_name : str\n        The name of the dataset.\n    function : str\n        The evaluation metric of this item (e.g., accuracy).\n    upload_time : str\n        The time of evaluation.\n    uploader: int\n        Uploader ID (user ID)\n    upload_name : str\n        Name of the uploader of this evaluation\n    value : float\n        The value (score) of this evaluation.\n    values : List[float]\n        The values (scores) per repeat and fold (if requested)\n    array_data : str\n        list of information per class.\n        (e.g., in case of precision, auroc, recall)\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        run_id: int,\n        task_id: int,\n        setup_id: int,\n        flow_id: int,\n        flow_name: str,\n        data_id: int,\n        data_name: str,\n        function: str,\n        upload_time: str,\n        uploader: int,\n        uploader_name: str,\n        value: float | None,\n        values: list[float] | None,\n        array_data: str | None = None,\n    ):\n        self.run_id = run_id\n        self.task_id = task_id\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.data_id = data_id\n        self.data_name = data_name\n        self.function = function\n        self.upload_time = upload_time\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.value = value\n        self.values = values\n        self.array_data = array_data\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Evaluation\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Upload Date\": self.upload_time,\n            \"Run ID\": self.run_id,\n            \"OpenML Run URL\": openml.runs.OpenMLRun.url_for_id(self.run_id),\n            \"Task ID\": self.task_id,\n            \"OpenML Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"OpenML Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Setup ID\": self.setup_id,\n            \"Data ID\": self.data_id,\n            \"Data Name\": self.data_name,\n            \"OpenML Data URL\": openml.datasets.OpenMLDataset.url_for_id(self.data_id),\n            \"Metric Used\": self.function,\n            \"Result\": self.value,\n        }\n\n        order = [\n            \"Uploader Date\",\n            \"Run ID\",\n            \"OpenML Run URL\",\n            \"Task ID\",\n            \"OpenML Task URL\" \"Flow ID\",\n            \"OpenML Flow URL\",\n            \"Setup ID\",\n            \"Data ID\",\n            \"Data Name\",\n            \"OpenML Data URL\",\n            \"Metric Used\",\n            \"Result\",\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow","title":"<code>OpenMLFlow</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Flow. Stores machine learning models.</p> <p>Flows should not be generated manually, but by the function :meth:<code>openml.flows.create_flow_from_model</code>. Using this helper function ensures that all relevant fields are filled in.</p> <p>Implements <code>openml.implementation.upload.xsd &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/ openml.implementation.upload.xsd&gt;</code>_.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the flow. Is used together with the attribute <code>external_version</code> as a unique identifier of the flow.</p> required <code>description</code> <code>str</code> <p>Human-readable description of the flow (free text).</p> required <code>model</code> <code>object</code> <p>ML model which is described by this flow.</p> required <code>components</code> <code>OrderedDict</code> <p>Mapping from component identifier to an OpenMLFlow object. Components are usually subfunctions of an algorithm (e.g. kernels), base learners in ensemble algorithms (decision tree in adaboost) or building blocks of a machine learning pipeline. Components are modeled as independent flows and can be shared between flows (different pipelines can use the same components).</p> required <code>parameters</code> <code>OrderedDict</code> <p>Mapping from parameter name to the parameter default value. The parameter default value must be of type <code>str</code>, so that the respective toolbox plugin can take care of casting the parameter default value to the correct type.</p> required <code>parameters_meta_info</code> <code>OrderedDict</code> <p>Mapping from parameter name to <code>dict</code>. Stores additional information for each parameter. Required keys are <code>data_type</code> and <code>description</code>.</p> required <code>external_version</code> <code>str</code> <p>Version number of the software the flow is implemented in. Is used together with the attribute <code>name</code> as a uniquer identifier of the flow.</p> required <code>tags</code> <code>list</code> <p>List of tags. Created on the server by other API calls.</p> required <code>language</code> <code>str</code> <p>Natural language the flow is described in (not the programming language).</p> required <code>dependencies</code> <code>str</code> <p>A list of dependencies necessary to run the flow. This field should contain all libraries the flow depends on. To allow reproducibility it should also specify the exact version numbers.</p> required <code>class_name</code> <code>str</code> <p>The development language name of the class which is described by this flow.</p> <code>None</code> <code>custom_name</code> <code>str</code> <p>Custom name of the flow given by the owner.</p> <code>None</code> <code>binary_url</code> <code>str</code> <p>Url from which the binary can be downloaded. Added by the server. Ignored when uploaded manually. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_format</code> <code>str</code> <p>Format in which the binary code was uploaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_md5</code> <code>str</code> <p>MD5 checksum to check if the binary code was correctly downloaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>uploader</code> <code>str</code> <p>OpenML user ID of the uploader. Filled in by the server.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>Date the flow was uploaded. Filled in by the server.</p> <code>None</code> <code>flow_id</code> <code>int</code> <p>Flow ID. Assigned by the server.</p> <code>None</code> <code>extension</code> <code>Extension</code> <p>The extension for a flow (e.g., sklearn).</p> <code>None</code> <code>version</code> <code>str</code> <p>OpenML version of the flow. Assigned by the server.</p> <code>None</code> Source code in <code>openml/flows/flow.py</code> <pre><code>class OpenMLFlow(OpenMLBase):\n    \"\"\"OpenML Flow. Stores machine learning models.\n\n    Flows should not be generated manually, but by the function\n    :meth:`openml.flows.create_flow_from_model`. Using this helper function\n    ensures that all relevant fields are filled in.\n\n    Implements `openml.implementation.upload.xsd\n    &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/\n    openml.implementation.upload.xsd&gt;`_.\n\n    Parameters\n    ----------\n    name : str\n        Name of the flow. Is used together with the attribute\n        `external_version` as a unique identifier of the flow.\n    description : str\n        Human-readable description of the flow (free text).\n    model : object\n        ML model which is described by this flow.\n    components : OrderedDict\n        Mapping from component identifier to an OpenMLFlow object. Components\n        are usually subfunctions of an algorithm (e.g. kernels), base learners\n        in ensemble algorithms (decision tree in adaboost) or building blocks\n        of a machine learning pipeline. Components are modeled as independent\n        flows and can be shared between flows (different pipelines can use\n        the same components).\n    parameters : OrderedDict\n        Mapping from parameter name to the parameter default value. The\n        parameter default value must be of type `str`, so that the respective\n        toolbox plugin can take care of casting the parameter default value to\n        the correct type.\n    parameters_meta_info : OrderedDict\n        Mapping from parameter name to `dict`. Stores additional information\n        for each parameter. Required keys are `data_type` and `description`.\n    external_version : str\n        Version number of the software the flow is implemented in. Is used\n        together with the attribute `name` as a uniquer identifier of the flow.\n    tags : list\n        List of tags. Created on the server by other API calls.\n    language : str\n        Natural language the flow is described in (not the programming\n        language).\n    dependencies : str\n        A list of dependencies necessary to run the flow. This field should\n        contain all libraries the flow depends on. To allow reproducibility\n        it should also specify the exact version numbers.\n    class_name : str, optional\n        The development language name of the class which is described by this\n        flow.\n    custom_name : str, optional\n        Custom name of the flow given by the owner.\n    binary_url : str, optional\n        Url from which the binary can be downloaded. Added by the server.\n        Ignored when uploaded manually. Will not be used by the python API\n        because binaries aren't compatible across machines.\n    binary_format : str, optional\n        Format in which the binary code was uploaded. Will not be used by the\n        python API because binaries aren't compatible across machines.\n    binary_md5 : str, optional\n        MD5 checksum to check if the binary code was correctly downloaded. Will\n        not be used by the python API because binaries aren't compatible across\n        machines.\n    uploader : str, optional\n        OpenML user ID of the uploader. Filled in by the server.\n    upload_date : str, optional\n        Date the flow was uploaded. Filled in by the server.\n    flow_id : int, optional\n        Flow ID. Assigned by the server.\n    extension : Extension, optional\n        The extension for a flow (e.g., sklearn).\n    version : str, optional\n        OpenML version of the flow. Assigned by the server.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        name: str,\n        description: str,\n        model: object,\n        components: dict,\n        parameters: dict,\n        parameters_meta_info: dict,\n        external_version: str,\n        tags: list,\n        language: str,\n        dependencies: str,\n        class_name: str | None = None,\n        custom_name: str | None = None,\n        binary_url: str | None = None,\n        binary_format: str | None = None,\n        binary_md5: str | None = None,\n        uploader: str | None = None,\n        upload_date: str | None = None,\n        flow_id: int | None = None,\n        extension: Extension | None = None,\n        version: str | None = None,\n    ):\n        self.name = name\n        self.description = description\n        self.model = model\n\n        for variable, variable_name in [\n            [components, \"components\"],\n            [parameters, \"parameters\"],\n            [parameters_meta_info, \"parameters_meta_info\"],\n        ]:\n            if not isinstance(variable, (OrderedDict, dict)):\n                raise TypeError(\n                    f\"{variable_name} must be of type OrderedDict or dict, \"\n                    f\"but is {type(variable)}.\",\n                )\n\n        self.components = components\n        self.parameters = parameters\n        self.parameters_meta_info = parameters_meta_info\n        self.class_name = class_name\n\n        keys_parameters = set(parameters.keys())\n        keys_parameters_meta_info = set(parameters_meta_info.keys())\n        if len(keys_parameters.difference(keys_parameters_meta_info)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters, but not in \"\n                \"parameters_meta_info.\"\n                % str(keys_parameters.difference(keys_parameters_meta_info)),\n            )\n        if len(keys_parameters_meta_info.difference(keys_parameters)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters_meta_info, \"\n                \"but not in parameters.\"\n                % str(keys_parameters_meta_info.difference(keys_parameters)),\n            )\n\n        self.external_version = external_version\n        self.uploader = uploader\n\n        self.custom_name = custom_name\n        self.tags = tags if tags is not None else []\n        self.binary_url = binary_url\n        self.binary_format = binary_format\n        self.binary_md5 = binary_md5\n        self.version = version\n        self.upload_date = upload_date\n        self.language = language\n        self.dependencies = dependencies\n        self.flow_id = flow_id\n        if extension is None:\n            self._extension = get_extension_by_flow(self)\n        else:\n            self._extension = extension\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the flow.\"\"\"\n        return self.flow_id\n\n    @property\n    def extension(self) -&gt; Extension:\n        \"\"\"The extension of the flow (e.g., sklearn).\"\"\"\n        if self._extension is not None:\n            return self._extension\n\n        raise RuntimeError(\n            f\"No extension could be found for flow {self.flow_id}: {self.name}\",\n        )\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields = {\n            \"Flow Name\": self.name,\n            \"Flow Description\": self.description,\n            \"Dependencies\": self.dependencies,\n        }\n        if self.flow_id is not None:\n            fields[\"Flow URL\"] = self.openml_url if self.openml_url is not None else \"None\"\n            fields[\"Flow ID\"] = str(self.flow_id)\n            if self.version is not None:\n                fields[\"Flow ID\"] += f\" (version {self.version})\"\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.binary_url is not None:\n            fields[\"Binary URL\"] = self.binary_url\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Flow ID\",\n            \"Flow URL\",\n            \"Flow Name\",\n            \"Flow Description\",\n            \"Binary URL\",\n            \"Upload Date\",\n            \"Dependencies\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: C901, PLR0912\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        flow_container = OrderedDict()  # type: 'dict[str, dict]'\n        flow_dict = OrderedDict(\n            [(\"@xmlns:oml\", \"http://openml.org/openml\")],\n        )  # type: 'dict[str, list | str]'  # E501\n        flow_container[\"oml:flow\"] = flow_dict\n        _add_if_nonempty(flow_dict, \"oml:id\", self.flow_id)\n\n        for required in [\"name\", \"external_version\"]:\n            if getattr(self, required) is None:\n                raise ValueError(f\"self.{required} is required but None\")\n        for attribute in [\n            \"uploader\",\n            \"name\",\n            \"custom_name\",\n            \"class_name\",\n            \"version\",\n            \"external_version\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n        ]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        if not self.description:\n            logger = logging.getLogger(__name__)\n            logger.warning(\"Flow % has empty description\", self.name)\n\n        flow_parameters = []\n        for key in self.parameters:\n            param_dict = OrderedDict()  # type: 'OrderedDict[str, str]'\n            param_dict[\"oml:name\"] = key\n            meta_info = self.parameters_meta_info[key]\n\n            _add_if_nonempty(param_dict, \"oml:data_type\", meta_info[\"data_type\"])\n            param_dict[\"oml:default_value\"] = self.parameters[key]\n            _add_if_nonempty(param_dict, \"oml:description\", meta_info[\"description\"])\n\n            for key_, value in param_dict.items():\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n                if value is not None and not isinstance(value, str):\n                    raise ValueError(\n                        f\"Parameter value {value} cannot be serialized \"\n                        f\"because it is of type {type(value)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            flow_parameters.append(param_dict)\n\n        flow_dict[\"oml:parameter\"] = flow_parameters\n\n        components = []\n        for key in self.components:\n            component_dict = OrderedDict()  # type: 'OrderedDict[str, dict]'\n            component_dict[\"oml:identifier\"] = key\n            if self.components[key] in [\"passthrough\", \"drop\"]:\n                component_dict[\"oml:flow\"] = {\n                    \"oml-python:serialized_object\": \"component_reference\",\n                    \"value\": {\"key\": self.components[key], \"step_name\": self.components[key]},\n                }\n            else:\n                component_dict[\"oml:flow\"] = self.components[key]._to_dict()[\"oml:flow\"]\n\n            for key_ in component_dict:\n                # We only need to check if the key is a string, because the\n                # value is a flow. The flow itself is valid by recursion\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            components.append(component_dict)\n\n        flow_dict[\"oml:component\"] = components\n        flow_dict[\"oml:tag\"] = self.tags\n        for attribute in [\"binary_url\", \"binary_format\", \"binary_md5\"]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        return flow_container\n\n    @classmethod\n    def _from_dict(cls, xml_dict: dict) -&gt; OpenMLFlow:\n        \"\"\"Create a flow from an xml description.\n\n        Calls itself recursively to create :class:`OpenMLFlow` objects of\n        subflows (components).\n\n        XML definition of a flow is available at\n        https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd\n\n        Parameters\n        ----------\n        xml_dict : dict\n            Dictionary representation of the flow as created by _to_dict()\n\n        Returns\n        -------\n            OpenMLFlow\n\n        \"\"\"  # E501\n        arguments = OrderedDict()\n        dic = xml_dict[\"oml:flow\"]\n\n        # Mandatory parts in the xml file\n        for key in [\"name\"]:\n            arguments[key] = dic[\"oml:\" + key]\n\n        # non-mandatory parts in the xml file\n        for key in [\n            \"external_version\",\n            \"uploader\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n            \"version\",\n            \"binary_url\",\n            \"binary_format\",\n            \"binary_md5\",\n            \"class_name\",\n            \"custom_name\",\n        ]:\n            arguments[key] = dic.get(\"oml:\" + key)\n\n        # has to be converted to an int if present and cannot parsed in the\n        # two loops above\n        arguments[\"flow_id\"] = int(dic[\"oml:id\"]) if dic.get(\"oml:id\") is not None else None\n\n        # Now parse parts of a flow which can occur multiple times like\n        # parameters, components (subflows) and tags. These can't be tackled\n        # in the loops above because xmltodict returns a dict if such an\n        # entity occurs once, and a list if it occurs multiple times.\n        # Furthermore, they must be treated differently, for example\n        # for components this method is called recursively and\n        # for parameters the actual information is split into two dictionaries\n        # for easier access in python.\n\n        parameters = OrderedDict()\n        parameters_meta_info = OrderedDict()\n        if \"oml:parameter\" in dic:\n            # In case of a single parameter, xmltodict returns a dictionary,\n            # otherwise a list.\n            oml_parameters = extract_xml_tags(\"oml:parameter\", dic, allow_none=False)\n\n            for oml_parameter in oml_parameters:\n                parameter_name = oml_parameter[\"oml:name\"]\n                default_value = oml_parameter[\"oml:default_value\"]\n                parameters[parameter_name] = default_value\n\n                meta_info = OrderedDict()\n                meta_info[\"description\"] = oml_parameter.get(\"oml:description\")\n                meta_info[\"data_type\"] = oml_parameter.get(\"oml:data_type\")\n                parameters_meta_info[parameter_name] = meta_info\n        arguments[\"parameters\"] = parameters\n        arguments[\"parameters_meta_info\"] = parameters_meta_info\n\n        components = OrderedDict()\n        if \"oml:component\" in dic:\n            # In case of a single component xmltodict returns a dict,\n            # otherwise a list.\n            oml_components = extract_xml_tags(\"oml:component\", dic, allow_none=False)\n\n            for component in oml_components:\n                flow = OpenMLFlow._from_dict(component)\n                components[component[\"oml:identifier\"]] = flow\n        arguments[\"components\"] = components\n        arguments[\"tags\"] = extract_xml_tags(\"oml:tag\", dic)\n\n        arguments[\"model\"] = None\n        return cls(**arguments)\n\n    def to_filesystem(self, output_directory: str | Path) -&gt; None:\n        \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n        output_directory = Path(output_directory)\n        output_directory.mkdir(parents=True, exist_ok=True)\n\n        output_path = output_directory / \"flow.xml\"\n        if output_path.exists():\n            raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n        run_xml = self._to_xml()\n        with output_path.open(\"w\") as f:\n            f.write(run_xml)\n\n    @classmethod\n    def from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n        \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n        input_directory = Path(input_directory) / \"flow.xml\"\n        with input_directory.open() as f:\n            xml_string = f.read()\n        return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.flow_id = int(xml_response[\"oml:upload_flow\"][\"oml:id\"])\n\n    def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n        \"\"\"Publish this flow to OpenML server.\n\n        Raises a PyOpenMLError if the flow exists on the server, but\n        `self.flow_id` does not match the server known flow id.\n\n        Parameters\n        ----------\n        raise_error_if_exists : bool, optional (default=False)\n            If True, raise PyOpenMLError if the flow exists on the server.\n            If False, update the local flow to match the server flow.\n\n        Returns\n        -------\n        self : OpenMLFlow\n\n        \"\"\"\n        # Import at top not possible because of cyclic dependencies. In\n        # particular, flow.py tries to import functions.py in order to call\n        # get_flow(), while functions.py tries to import flow.py in order to\n        # instantiate an OpenMLFlow.\n        import openml.flows.functions\n\n        flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n        if not flow_id:\n            if self.flow_id:\n                raise openml.exceptions.PyOpenMLError(\n                    \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n                )\n            super().publish()\n            assert self.flow_id is not None  # for mypy\n            flow_id = self.flow_id\n        elif raise_error_if_exists:\n            error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n            raise openml.exceptions.PyOpenMLError(error_message)\n        elif self.flow_id is not None and self.flow_id != flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n            )\n\n        flow = openml.flows.functions.get_flow(flow_id)\n        _copy_server_fields(flow, self)\n        try:\n            openml.flows.functions.assert_flows_equal(\n                self,\n                flow,\n                flow.upload_date,\n                ignore_parameter_values=True,\n                ignore_custom_name_if_none=True,\n            )\n        except ValueError as e:\n            message = e.args[0]\n            raise ValueError(\n                \"The flow on the server is inconsistent with the local flow. \"\n                f\"The server flow ID is {flow_id}. Please check manually and remove \"\n                f\"the flow if necessary! Error is:\\n'{message}'\",\n            ) from e\n        return self\n\n    def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n        \"\"\"\n        Returns for each sub-component of the flow the path of identifiers\n        that should be traversed to reach this component. The resulting dict\n        maps a key (identifying a flow by either its id, name or fullname) to\n        the parameter prefix.\n\n        Parameters\n        ----------\n        key_item: str\n            The flow attribute that will be used to identify flows in the\n            structure. Allowed values {flow_id, name}\n\n        Returns\n        -------\n        dict[str, List[str]]\n            The flow structure\n        \"\"\"\n        if key_item not in [\"flow_id\", \"name\"]:\n            raise ValueError(\"key_item should be in {flow_id, name}\")\n        structure = {}\n        for key, sub_flow in self.components.items():\n            sub_structure = sub_flow.get_structure(key_item)\n            for flow_name, flow_sub_structure in sub_structure.items():\n                structure[flow_name] = [key, *flow_sub_structure]\n        structure[getattr(self, key_item)] = []\n        return structure\n\n    def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n        \"\"\"\n        Returns a subflow from the tree of dependencies.\n\n        Parameters\n        ----------\n        structure: list[str]\n            A list of strings, indicating the location of the subflow\n\n        Returns\n        -------\n        OpenMLFlow\n            The OpenMLFlow that corresponds to the structure\n        \"\"\"\n        # make a copy of structure, as we don't want to change it in the\n        # outer scope\n        structure = list(structure)\n        if len(structure) &lt; 1:\n            raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n        sub_identifier = structure[0]\n        if sub_identifier not in self.components:\n            raise ValueError(\n                f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n            )\n        if len(structure) == 1:\n            return self.components[sub_identifier]  # type: ignore\n\n        structure.pop(0)\n        return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.extension","title":"<code>extension: Extension</code>  <code>property</code>","text":"<p>The extension of the flow (e.g., sklearn).</p>"},{"location":"reference/#openml.OpenMLFlow.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the flow.</p>"},{"location":"reference/#openml.OpenMLFlow.from_filesystem","title":"<code>from_filesystem(input_directory)</code>  <code>classmethod</code>","text":"<p>Read a flow from an XML in input_directory on the filesystem.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n    \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n    input_directory = Path(input_directory) / \"flow.xml\"\n    with input_directory.open() as f:\n        xml_string = f.read()\n    return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.get_structure","title":"<code>get_structure(key_item)</code>","text":"<p>Returns for each sub-component of the flow the path of identifiers that should be traversed to reach this component. The resulting dict maps a key (identifying a flow by either its id, name or fullname) to the parameter prefix.</p> <p>Parameters:</p> Name Type Description Default <code>key_item</code> <code>str</code> <p>The flow attribute that will be used to identify flows in the structure. Allowed values {flow_id, name}</p> required <p>Returns:</p> Type Description <code>dict[str, List[str]]</code> <p>The flow structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n    \"\"\"\n    Returns for each sub-component of the flow the path of identifiers\n    that should be traversed to reach this component. The resulting dict\n    maps a key (identifying a flow by either its id, name or fullname) to\n    the parameter prefix.\n\n    Parameters\n    ----------\n    key_item: str\n        The flow attribute that will be used to identify flows in the\n        structure. Allowed values {flow_id, name}\n\n    Returns\n    -------\n    dict[str, List[str]]\n        The flow structure\n    \"\"\"\n    if key_item not in [\"flow_id\", \"name\"]:\n        raise ValueError(\"key_item should be in {flow_id, name}\")\n    structure = {}\n    for key, sub_flow in self.components.items():\n        sub_structure = sub_flow.get_structure(key_item)\n        for flow_name, flow_sub_structure in sub_structure.items():\n            structure[flow_name] = [key, *flow_sub_structure]\n    structure[getattr(self, key_item)] = []\n    return structure\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.get_subflow","title":"<code>get_subflow(structure)</code>","text":"<p>Returns a subflow from the tree of dependencies.</p> <p>Parameters:</p> Name Type Description Default <code>structure</code> <code>list[str]</code> <p>A list of strings, indicating the location of the subflow</p> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> <p>The OpenMLFlow that corresponds to the structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n    \"\"\"\n    Returns a subflow from the tree of dependencies.\n\n    Parameters\n    ----------\n    structure: list[str]\n        A list of strings, indicating the location of the subflow\n\n    Returns\n    -------\n    OpenMLFlow\n        The OpenMLFlow that corresponds to the structure\n    \"\"\"\n    # make a copy of structure, as we don't want to change it in the\n    # outer scope\n    structure = list(structure)\n    if len(structure) &lt; 1:\n        raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n    sub_identifier = structure[0]\n    if sub_identifier not in self.components:\n        raise ValueError(\n            f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n        )\n    if len(structure) == 1:\n        return self.components[sub_identifier]  # type: ignore\n\n    structure.pop(0)\n    return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.publish","title":"<code>publish(raise_error_if_exists=False)</code>","text":"<p>Publish this flow to OpenML server.</p> <p>Raises a PyOpenMLError if the flow exists on the server, but <code>self.flow_id</code> does not match the server known flow id.</p> <p>Parameters:</p> Name Type Description Default <code>raise_error_if_exists</code> <code>(bool, optional(default=False))</code> <p>If True, raise PyOpenMLError if the flow exists on the server. If False, update the local flow to match the server flow.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>self</code> <code>OpenMLFlow</code> Source code in <code>openml/flows/flow.py</code> <pre><code>def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Publish this flow to OpenML server.\n\n    Raises a PyOpenMLError if the flow exists on the server, but\n    `self.flow_id` does not match the server known flow id.\n\n    Parameters\n    ----------\n    raise_error_if_exists : bool, optional (default=False)\n        If True, raise PyOpenMLError if the flow exists on the server.\n        If False, update the local flow to match the server flow.\n\n    Returns\n    -------\n    self : OpenMLFlow\n\n    \"\"\"\n    # Import at top not possible because of cyclic dependencies. In\n    # particular, flow.py tries to import functions.py in order to call\n    # get_flow(), while functions.py tries to import flow.py in order to\n    # instantiate an OpenMLFlow.\n    import openml.flows.functions\n\n    flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n    if not flow_id:\n        if self.flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n            )\n        super().publish()\n        assert self.flow_id is not None  # for mypy\n        flow_id = self.flow_id\n    elif raise_error_if_exists:\n        error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n        raise openml.exceptions.PyOpenMLError(error_message)\n    elif self.flow_id is not None and self.flow_id != flow_id:\n        raise openml.exceptions.PyOpenMLError(\n            \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n        )\n\n    flow = openml.flows.functions.get_flow(flow_id)\n    _copy_server_fields(flow, self)\n    try:\n        openml.flows.functions.assert_flows_equal(\n            self,\n            flow,\n            flow.upload_date,\n            ignore_parameter_values=True,\n            ignore_custom_name_if_none=True,\n        )\n    except ValueError as e:\n        message = e.args[0]\n        raise ValueError(\n            \"The flow on the server is inconsistent with the local flow. \"\n            f\"The server flow ID is {flow_id}. Please check manually and remove \"\n            f\"the flow if necessary! Error is:\\n'{message}'\",\n        ) from e\n    return self\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.to_filesystem","title":"<code>to_filesystem(output_directory)</code>","text":"<p>Write a flow to the filesystem as XML to output_directory.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def to_filesystem(self, output_directory: str | Path) -&gt; None:\n    \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n    output_directory = Path(output_directory)\n    output_directory.mkdir(parents=True, exist_ok=True)\n\n    output_path = output_directory / \"flow.xml\"\n    if output_path.exists():\n        raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n    run_xml = self._to_xml()\n    with output_path.open(\"w\") as f:\n        f.write(run_xml)\n</code></pre>"},{"location":"reference/#openml.OpenMLLearningCurveTask","title":"<code>OpenMLLearningCurveTask</code>","text":"<p>               Bases: <code>OpenMLClassificationTask</code></p> <p>OpenML Learning Curve object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Learning Curve task.</p> required <code>task_type</code> <code>str</code> <p>Name of the Learning Curve task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the dataset that this task is associated with.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature in the dataset.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure to use for evaluating models.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Additional parameters for the estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the file containing the data splits for Learning Curve task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Learning Curve task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure to use for evaluating models.</p> <code>None</code> <code>class_labels</code> <code>list of str</code> <p>Class labels for Learning Curve tasks.</p> <code>None</code> <code>cost_matrix</code> <code>numpy array</code> <p>Cost matrix for Learning Curve tasks.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLLearningCurveTask(OpenMLClassificationTask):\n    \"\"\"OpenML Learning Curve object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Learning Curve task.\n    task_type : str\n        Name of the Learning Curve task.\n    data_set_id : int\n        ID of the dataset that this task is associated with.\n    target_name : str\n        Name of the target feature in the dataset.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure to use for evaluating models.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Additional parameters for the estimation procedure.\n    data_splits_url : str, default=None\n        URL of the file containing the data splits for Learning Curve task.\n    task_id : Union[int, None]\n        ID of the Learning Curve task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure to use for evaluating models.\n    class_labels : list of str, default=None\n        Class labels for Learning Curve tasks.\n    cost_matrix : numpy array, default=None\n        Cost matrix for Learning Curve tasks.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 13,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n            class_labels=class_labels,\n            cost_matrix=cost_matrix,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLParameter","title":"<code>OpenMLParameter</code>","text":"<p>Parameter object (used in setup).</p> <p>Parameters:</p> Name Type Description Default <code>input_id</code> <code>int</code> <p>The input id from the openml database</p> required <code>flow</code> <p>The flow to which this parameter is associated</p> required <code>flow</code> <p>The name of the flow (no version number) to which this parameter is associated</p> required <code>full_name</code> <code>str</code> <p>The name of the flow and parameter combined</p> required <code>parameter_name</code> <code>str</code> <p>The name of the parameter</p> required <code>data_type</code> <code>str</code> <p>The datatype of the parameter. generally unused for sklearn flows</p> required <code>default_value</code> <code>str</code> <p>The default value. For sklearn parameters, this is unknown and a default value is selected arbitrarily</p> required <code>value</code> <code>str</code> <p>If the parameter was set, the value that it was set to.</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLParameter:\n    \"\"\"Parameter object (used in setup).\n\n    Parameters\n    ----------\n    input_id : int\n        The input id from the openml database\n    flow id : int\n        The flow to which this parameter is associated\n    flow name : str\n        The name of the flow (no version number) to which this parameter\n        is associated\n    full_name : str\n        The name of the flow and parameter combined\n    parameter_name : str\n        The name of the parameter\n    data_type : str\n        The datatype of the parameter. generally unused for sklearn flows\n    default_value : str\n        The default value. For sklearn parameters, this is unknown and a\n        default value is selected arbitrarily\n    value : str\n        If the parameter was set, the value that it was set to.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        input_id: int,\n        flow_id: int,\n        flow_name: str,\n        full_name: str,\n        parameter_name: str,\n        data_type: str,\n        default_value: str,\n        value: str,\n    ):\n        self.id = input_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.full_name = full_name\n        self.parameter_name = parameter_name\n        self.data_type = data_type\n        self.default_value = default_value\n        self.value = value\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Parameter\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"ID\": self.id,\n            \"Flow ID\": self.flow_id,\n            # \"Flow Name\": self.flow_name,\n            \"Flow Name\": self.full_name,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Parameter Name\": self.parameter_name,\n        }\n        # indented prints for parameter attributes\n        # indention = 2 spaces + 1 | + 2 underscores\n        indent = \"{}|{}\".format(\" \" * 2, \"_\" * 2)\n        parameter_data_type = f\"{indent}Data Type\"\n        fields[parameter_data_type] = self.data_type\n        parameter_default = f\"{indent}Default\"\n        fields[parameter_default] = self.default_value\n        parameter_value = f\"{indent}Value\"\n        fields[parameter_value] = self.value\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Parameter Name\",\n            parameter_data_type,\n            parameter_default,\n            parameter_value,\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/#openml.OpenMLRegressionTask","title":"<code>OpenMLRegressionTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Regression object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML Regression task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML Regression task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature used in the Regression task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the Regression task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML Regression task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the Regression task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLRegressionTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Regression object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML Regression task.\n    task_type : str\n        Task type of the OpenML Regression task.\n    data_set_id : int\n        ID of the OpenML dataset.\n    target_name : str\n        Name of the target feature used in the Regression task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the Regression task.\n    task_id : Union[int, None]\n        ID of the OpenML Regression task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the Regression task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 7,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLRun","title":"<code>OpenMLRun</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Run: result of running a model on an OpenML dataset.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The ID of the OpenML task associated with the run.</p> required <code>flow_id</code> <code>int | None</code> <p>The ID of the OpenML flow associated with the run.</p> required <code>dataset_id</code> <code>int | None</code> <p>The ID of the OpenML dataset used for the run.</p> required <code>setup_string</code> <code>str | None</code> <p>The setup string of the run.</p> <code>None</code> <code>output_files</code> <code>dict[str, int] | None</code> <p>Specifies where each related file can be found.</p> <code>None</code> <code>setup_id</code> <code>int | None</code> <p>An integer representing the ID of the setup used for the run.</p> <code>None</code> <code>tags</code> <code>list[str] | None</code> <p>Representing the tags associated with the run.</p> <code>None</code> <code>uploader</code> <code>int | None</code> <p>User ID of the uploader.</p> <code>None</code> <code>uploader_name</code> <code>str | None</code> <p>The name of the person who uploaded the run.</p> <code>None</code> <code>evaluations</code> <code>dict | None</code> <p>Representing the evaluations of the run.</p> <code>None</code> <code>fold_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each fold.</p> <code>None</code> <code>sample_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each sample.</p> <code>None</code> <code>data_content</code> <code>list[list] | None</code> <p>The predictions generated from executing this run.</p> <code>None</code> <code>trace</code> <code>OpenMLRunTrace | None</code> <p>The trace containing information on internal model evaluations of this run.</p> <code>None</code> <code>model</code> <code>object | None</code> <p>The untrained model that was evaluated in the run.</p> <code>None</code> <code>task_type</code> <code>str | None</code> <p>The type of the OpenML task associated with the run.</p> <code>None</code> <code>task_evaluation_measure</code> <code>str | None</code> <p>The evaluation measure used for the task.</p> <code>None</code> <code>flow_name</code> <code>str | None</code> <p>The name of the OpenML flow associated with the run.</p> <code>None</code> <code>parameter_settings</code> <code>list[dict[str, Any]] | None</code> <p>Representing the parameter settings used for the run.</p> <code>None</code> <code>predictions_url</code> <code>str | None</code> <p>The URL of the predictions file.</p> <code>None</code> <code>task</code> <code>OpenMLTask | None</code> <p>An instance of the OpenMLTask class, representing the OpenML task associated with the run.</p> <code>None</code> <code>flow</code> <code>OpenMLFlow | None</code> <p>An instance of the OpenMLFlow class, representing the OpenML flow associated with the run.</p> <code>None</code> <code>run_id</code> <code>int | None</code> <p>The ID of the run.</p> <code>None</code> <code>description_text</code> <code>str | None</code> <p>Description text to add to the predictions file. If left None, is set to the time the arff file is generated.</p> <code>None</code> <code>run_details</code> <code>str | None</code> <p>Description of the run stored in the run meta-data.</p> <code>None</code> Source code in <code>openml/runs/run.py</code> <pre><code>class OpenMLRun(OpenMLBase):\n    \"\"\"OpenML Run: result of running a model on an OpenML dataset.\n\n    Parameters\n    ----------\n    task_id: int\n        The ID of the OpenML task associated with the run.\n    flow_id: int\n        The ID of the OpenML flow associated with the run.\n    dataset_id: int\n        The ID of the OpenML dataset used for the run.\n    setup_string: str\n        The setup string of the run.\n    output_files: Dict[str, int]\n        Specifies where each related file can be found.\n    setup_id: int\n        An integer representing the ID of the setup used for the run.\n    tags: List[str]\n        Representing the tags associated with the run.\n    uploader: int\n        User ID of the uploader.\n    uploader_name: str\n        The name of the person who uploaded the run.\n    evaluations: Dict\n        Representing the evaluations of the run.\n    fold_evaluations: Dict\n        The evaluations of the run for each fold.\n    sample_evaluations: Dict\n        The evaluations of the run for each sample.\n    data_content: List[List]\n        The predictions generated from executing this run.\n    trace: OpenMLRunTrace\n        The trace containing information on internal model evaluations of this run.\n    model: object\n        The untrained model that was evaluated in the run.\n    task_type: str\n        The type of the OpenML task associated with the run.\n    task_evaluation_measure: str\n        The evaluation measure used for the task.\n    flow_name: str\n        The name of the OpenML flow associated with the run.\n    parameter_settings: list[OrderedDict]\n        Representing the parameter settings used for the run.\n    predictions_url: str\n        The URL of the predictions file.\n    task: OpenMLTask\n        An instance of the OpenMLTask class, representing the OpenML task associated\n        with the run.\n    flow: OpenMLFlow\n        An instance of the OpenMLFlow class, representing the OpenML flow associated\n        with the run.\n    run_id: int\n        The ID of the run.\n    description_text: str, optional\n        Description text to add to the predictions file. If left None, is set to the\n        time the arff file is generated.\n    run_details: str, optional (default=None)\n        Description of the run stored in the run meta-data.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int,\n        flow_id: int | None,\n        dataset_id: int | None,\n        setup_string: str | None = None,\n        output_files: dict[str, int] | None = None,\n        setup_id: int | None = None,\n        tags: list[str] | None = None,\n        uploader: int | None = None,\n        uploader_name: str | None = None,\n        evaluations: dict | None = None,\n        fold_evaluations: dict | None = None,\n        sample_evaluations: dict | None = None,\n        data_content: list[list] | None = None,\n        trace: OpenMLRunTrace | None = None,\n        model: object | None = None,\n        task_type: str | None = None,\n        task_evaluation_measure: str | None = None,\n        flow_name: str | None = None,\n        parameter_settings: list[dict[str, Any]] | None = None,\n        predictions_url: str | None = None,\n        task: OpenMLTask | None = None,\n        flow: OpenMLFlow | None = None,\n        run_id: int | None = None,\n        description_text: str | None = None,\n        run_details: str | None = None,\n    ):\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.task_id = task_id\n        self.task_type = task_type\n        self.task_evaluation_measure = task_evaluation_measure\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.setup_id = setup_id\n        self.setup_string = setup_string\n        self.parameter_settings = parameter_settings\n        self.dataset_id = dataset_id\n        self.evaluations = evaluations\n        self.fold_evaluations = fold_evaluations\n        self.sample_evaluations = sample_evaluations\n        self.data_content = data_content\n        self.output_files = output_files\n        self.trace = trace\n        self.error_message = None\n        self.task = task\n        self.flow = flow\n        self.run_id = run_id\n        self.model = model\n        self.tags = tags\n        self.predictions_url = predictions_url\n        self.description_text = description_text\n        self.run_details = run_details\n        self._predictions = None\n\n    @property\n    def predictions(self) -&gt; pd.DataFrame:\n        \"\"\"Return a DataFrame with predictions for this run\"\"\"\n        if self._predictions is None:\n            if self.data_content:\n                arff_dict = self._generate_arff_dict()\n            elif self.predictions_url:\n                arff_text = openml._api_calls._download_text_file(self.predictions_url)\n                arff_dict = arff.loads(arff_text)\n            else:\n                raise RuntimeError(\"Run has no predictions.\")\n            self._predictions = pd.DataFrame(\n                arff_dict[\"data\"],\n                columns=[name for name, _ in arff_dict[\"attributes\"]],\n            )\n        return self._predictions\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the run, None if not uploaded to the server yet.\"\"\"\n        return self.run_id\n\n    def _evaluation_summary(self, metric: str) -&gt; str:\n        \"\"\"Summarizes the evaluation of a metric over all folds.\n\n        The fold scores for the metric must exist already. During run creation,\n        by default, the MAE for OpenMLRegressionTask and the accuracy for\n        OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed.\n\n        If repetition exist, we take the mean over all repetitions.\n\n        Parameters\n        ----------\n        metric: str\n            Name of an evaluation metric that was used to compute fold scores.\n\n        Returns\n        -------\n        metric_summary: str\n            A formatted string that displays the metric's evaluation summary.\n            The summary consists of the mean and std.\n        \"\"\"\n        if self.fold_evaluations is None:\n            raise ValueError(\"No fold evaluations available.\")\n        fold_score_lists = self.fold_evaluations[metric].values()\n\n        # Get the mean and std over all repetitions\n        rep_means = [np.mean(list(x.values())) for x in fold_score_lists]\n        rep_stds = [np.std(list(x.values())) for x in fold_score_lists]\n\n        return f\"{np.mean(rep_means):.4f} +- {np.mean(rep_stds):.4f}\"\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Set up fields\n        fields = {\n            \"Uploader Name\": self.uploader_name,\n            \"Metric\": self.task_evaluation_measure,\n            \"Run ID\": self.run_id,\n            \"Task ID\": self.task_id,\n            \"Task Type\": self.task_type,\n            \"Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"Flow Name\": self.flow_name,\n            \"Flow URL\": (\n                openml.flows.OpenMLFlow.url_for_id(self.flow_id)\n                if self.flow_id is not None\n                else None\n            ),\n            \"Setup ID\": self.setup_id,\n            \"Setup String\": self.setup_string,\n            \"Dataset ID\": self.dataset_id,\n            \"Dataset URL\": (\n                openml.datasets.OpenMLDataset.url_for_id(self.dataset_id)\n                if self.dataset_id is not None\n                else None\n            ),\n        }\n\n        # determines the order of the initial fields in which the information will be printed\n        order = [\"Uploader Name\", \"Uploader Profile\", \"Metric\", \"Result\"]\n\n        if self.uploader is not None:\n            fields[\"Uploader Profile\"] = f\"{openml.config.get_server_base_url()}/u/{self.uploader}\"\n        if self.run_id is not None:\n            fields[\"Run URL\"] = self.openml_url\n        if self.evaluations is not None and self.task_evaluation_measure in self.evaluations:\n            fields[\"Result\"] = self.evaluations[self.task_evaluation_measure]\n        elif self.fold_evaluations is not None:\n            # -- Add locally computed summary values if possible\n            if \"predictive_accuracy\" in self.fold_evaluations:\n                # OpenMLClassificationTask; OpenMLLearningCurveTask\n                result_field = \"Local Result - Accuracy (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"predictive_accuracy\")\n                order.append(result_field)\n            elif \"mean_absolute_error\" in self.fold_evaluations:\n                # OpenMLRegressionTask\n                result_field = \"Local Result - MAE (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"mean_absolute_error\")\n                order.append(result_field)\n\n            if \"usercpu_time_millis\" in self.fold_evaluations:\n                # Runtime should be available for most tasks types\n                rt_field = \"Local Runtime - ms (+- STD)\"\n                fields[rt_field] = self._evaluation_summary(\"usercpu_time_millis\")\n                order.append(rt_field)\n\n        # determines the remaining order\n        order += [\n            \"Run ID\",\n            \"Run URL\",\n            \"Task ID\",\n            \"Task Type\",\n            \"Task URL\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Setup ID\",\n            \"Setup String\",\n            \"Dataset ID\",\n            \"Dataset URL\",\n        ]\n        return [\n            (key, \"None\" if fields[key] is None else fields[key])  # type: ignore\n            for key in order\n            if key in fields\n        ]\n\n    @classmethod\n    def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n        \"\"\"\n        The inverse of the to_filesystem method. Instantiates an OpenMLRun\n        object based on files stored on the file system.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            are stored\n\n        expect_model : bool\n            if True, it requires the model pickle to be present, and an error\n            will be thrown if not. Otherwise, the model might or might not\n            be present.\n\n        Returns\n        -------\n        run : OpenMLRun\n            the re-instantiated run object\n        \"\"\"\n        # Avoiding cyclic imports\n        import openml.runs.functions\n\n        directory = Path(directory)\n        if not directory.is_dir():\n            raise ValueError(\"Could not find folder\")\n\n        description_path = directory / \"description.xml\"\n        predictions_path = directory / \"predictions.arff\"\n        trace_path = directory / \"trace.arff\"\n        model_path = directory / \"model.pkl\"\n\n        if not description_path.is_file():\n            raise ValueError(\"Could not find description.xml\")\n        if not predictions_path.is_file():\n            raise ValueError(\"Could not find predictions.arff\")\n        if (not model_path.is_file()) and expect_model:\n            raise ValueError(\"Could not find model.pkl\")\n\n        with description_path.open() as fht:\n            xml_string = fht.read()\n        run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n        if run.flow_id is None:\n            flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n            run.flow = flow\n            run.flow_name = flow.name\n\n        with predictions_path.open() as fht:\n            predictions = arff.load(fht)\n            run.data_content = predictions[\"data\"]\n\n        if model_path.is_file():\n            # note that it will load the model if the file exists, even if\n            # expect_model is False\n            with model_path.open(\"rb\") as fhb:\n                run.model = pickle.load(fhb)  # noqa: S301\n\n        if trace_path.is_file():\n            run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n        return run\n\n    def to_filesystem(\n        self,\n        directory: str | Path,\n        store_model: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        \"\"\"\n        The inverse of the from_filesystem method. Serializes a run\n        on the filesystem, to be uploaded later.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            will be stored. Should be empty\n\n        store_model : bool, optional (default=True)\n            if True, a model will be pickled as well. As this is the most\n            storage expensive part, it is often desirable to not store the\n            model.\n        \"\"\"\n        if self.data_content is None or self.model is None:\n            raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n        directory = Path(directory)\n        directory.mkdir(exist_ok=True, parents=True)\n\n        if any(directory.iterdir()):\n            raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n        run_xml = self._to_xml()\n        predictions_arff = arff.dumps(self._generate_arff_dict())\n\n        # It seems like typing does not allow to define the same variable multiple times\n        with (directory / \"description.xml\").open(\"w\") as fh:\n            fh.write(run_xml)\n        with (directory / \"predictions.arff\").open(\"w\") as fh:\n            fh.write(predictions_arff)\n        if store_model:\n            with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n                pickle.dump(self.model, fh_b)\n\n        if self.flow_id is None and self.flow is not None:\n            self.flow.to_filesystem(directory)\n\n        if self.trace is not None:\n            self.trace._to_filesystem(directory)\n\n    def _generate_arff_dict(self) -&gt; OrderedDict[str, Any]:\n        \"\"\"Generates the arff dictionary for uploading predictions to the\n        server.\n\n        Assumes that the run has been executed.\n\n        The order of the attributes follows the order defined by the Client API for R.\n\n        Returns\n        -------\n        arf_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains predictions and information about the run environment.\n        \"\"\"\n        if self.data_content is None:\n            raise ValueError(\"Run has not been executed.\")\n        if self.flow is None:\n            assert self.flow_id is not None, \"Run has no associated flow id!\"\n            self.flow = get_flow(self.flow_id)\n\n        if self.description_text is None:\n            self.description_text = time.strftime(\"%c\")\n        task = get_task(self.task_id)\n\n        arff_dict = OrderedDict()  # type: 'OrderedDict[str, Any]'\n        arff_dict[\"data\"] = self.data_content\n        arff_dict[\"description\"] = self.description_text\n        arff_dict[\"relation\"] = f\"openml_task_{task.task_id}_predictions\"\n\n        if isinstance(task, OpenMLLearningCurveTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"]\n                    + [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                    + [\n                        (\"confidence.\" + class_labels[i], \"NUMERIC\")\n                        for i in range(len(class_labels))\n                    ]\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLClassificationTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),  # Legacy\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                prediction_confidences = [\n                    (\"confidence.\" + class_labels[i], \"NUMERIC\") for i in range(len(class_labels))\n                ]\n                prediction_and_true = [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"] + prediction_and_true + prediction_confidences\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"prediction\", \"NUMERIC\"),\n                (\"truth\", \"NUMERIC\"),\n            ]\n\n        elif isinstance(task, OpenMLClusteringTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"cluster\", \"NUMERIC\"),\n            ]\n\n        else:\n            raise NotImplementedError(\"Task type %s is not yet supported.\" % str(task.task_type))\n\n        return arff_dict\n\n    def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n        \"\"\"Calculates metric scores based on predicted values. Assumes the\n        run has been executed locally (and contains run_data). Furthermore,\n        it assumes that the 'correct' or 'truth' attribute is specified in\n        the arff (which is an optional field, but always the case for\n        openml-python runs)\n\n        Parameters\n        ----------\n        sklearn_fn : function\n            a function pointer to a sklearn function that\n            accepts ``y_true``, ``y_pred`` and ``**kwargs``\n        kwargs : dict\n            kwargs for the function\n\n        Returns\n        -------\n        scores : ndarray of scores of length num_folds * num_repeats\n            metric results\n        \"\"\"\n        kwargs = kwargs if kwargs else {}\n        if self.data_content is not None and self.task_id is not None:\n            predictions_arff = self._generate_arff_dict()\n        elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n            predictions_file_url = openml._api_calls._file_id_to_url(\n                self.output_files[\"predictions\"],\n                \"predictions.arff\",\n            )\n            response = openml._api_calls._download_text_file(predictions_file_url)\n            predictions_arff = arff.loads(response)\n            # TODO: make this a stream reader\n        else:\n            raise ValueError(\n                \"Run should have been locally executed or \" \"contain outputfile reference.\",\n            )\n\n        # Need to know more about the task to compute scores correctly\n        task = get_task(self.task_id)\n\n        attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n        if (\n            task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n            and \"correct\" not in attribute_names\n        ):\n            raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n        if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n            raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n        if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n            raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n        def _attribute_list_to_dict(attribute_list):  # type: ignore\n            # convenience function: Creates a mapping to map from the name of\n            # attributes present in the arff prediction file to their index.\n            # This is necessary because the number of classes can be different\n            # for different tasks.\n            res = OrderedDict()\n            for idx in range(len(attribute_list)):\n                res[attribute_list[idx][0]] = idx\n            return res\n\n        attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n        repeat_idx = attribute_dict[\"repeat\"]\n        fold_idx = attribute_dict[\"fold\"]\n        predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n        if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n            correct_idx = attribute_dict[\"correct\"]\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            correct_idx = attribute_dict[\"truth\"]\n        has_samples = False\n        if \"sample\" in attribute_dict:\n            sample_idx = attribute_dict[\"sample\"]\n            has_samples = True\n\n        if (\n            predictions_arff[\"attributes\"][predicted_idx][1]\n            != predictions_arff[\"attributes\"][correct_idx][1]\n        ):\n            pred = predictions_arff[\"attributes\"][predicted_idx][1]\n            corr = predictions_arff[\"attributes\"][correct_idx][1]\n            raise ValueError(\n                \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n            )\n\n        # TODO: these could be cached\n        values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n        values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n        for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n            rep = line[repeat_idx]\n            fold = line[fold_idx]\n            samp = line[sample_idx] if has_samples else 0\n\n            if task.task_type_id in [\n                TaskType.SUPERVISED_CLASSIFICATION,\n                TaskType.LEARNING_CURVE,\n            ]:\n                prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                    line[predicted_idx],\n                )\n                correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n            elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n                prediction = line[predicted_idx]\n                correct = line[correct_idx]\n            if rep not in values_predict:\n                values_predict[rep] = OrderedDict()\n                values_correct[rep] = OrderedDict()\n            if fold not in values_predict[rep]:\n                values_predict[rep][fold] = OrderedDict()\n                values_correct[rep][fold] = OrderedDict()\n            if samp not in values_predict[rep][fold]:\n                values_predict[rep][fold][samp] = []\n                values_correct[rep][fold][samp] = []\n\n            values_predict[rep][fold][samp].append(prediction)\n            values_correct[rep][fold][samp].append(correct)\n\n        scores = []\n        for rep in values_predict:\n            for fold in values_predict[rep]:\n                last_sample = len(values_predict[rep][fold]) - 1\n                y_pred = values_predict[rep][fold][last_sample]\n                y_true = values_correct[rep][fold][last_sample]\n                scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n        return np.array(scores)\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.run_id = int(xml_response[\"oml:upload_run\"][\"oml:run_id\"])\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Get file_elements to upload to the server.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        if self.parameter_settings is None and self.model is None:\n            raise PyOpenMLError(\n                \"OpenMLRun must contain a model or be initialized with parameter_settings.\",\n            )\n        if self.flow_id is None:\n            if self.flow is None:\n                raise PyOpenMLError(\n                    \"OpenMLRun object does not contain a flow id or reference to OpenMLFlow \"\n                    \"(these should have been added while executing the task). \",\n                )\n\n            # publish the linked Flow before publishing the run.\n            self.flow.publish()\n            self.flow_id = self.flow.flow_id\n\n        if self.parameter_settings is None:\n            if self.flow is None:\n                assert self.flow_id is not None  # for mypy\n                self.flow = openml.flows.get_flow(self.flow_id)\n            self.parameter_settings = self.flow.extension.obtain_parameter_values(\n                self.flow,\n                self.model,\n            )\n\n        file_elements = {\"description\": (\"description.xml\", self._to_xml())}\n\n        if self.error_message is None:\n            predictions = arff.dumps(self._generate_arff_dict())\n            file_elements[\"predictions\"] = (\"predictions.arff\", predictions)\n\n        if self.trace is not None:\n            trace_arff = arff.dumps(self.trace.trace_to_arff())\n            file_elements[\"trace\"] = (\"trace.arff\", trace_arff)\n        return file_elements\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: PLR0912, C901\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        description = OrderedDict()  # type: 'OrderedDict'\n        description[\"oml:run\"] = OrderedDict()\n        description[\"oml:run\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n        description[\"oml:run\"][\"oml:task_id\"] = self.task_id\n        description[\"oml:run\"][\"oml:flow_id\"] = self.flow_id\n        if self.setup_string is not None:\n            description[\"oml:run\"][\"oml:setup_string\"] = self.setup_string\n        if self.error_message is not None:\n            description[\"oml:run\"][\"oml:error_message\"] = self.error_message\n        if self.run_details is not None:\n            description[\"oml:run\"][\"oml:run_details\"] = self.run_details\n        description[\"oml:run\"][\"oml:parameter_setting\"] = self.parameter_settings\n        if self.tags is not None:\n            description[\"oml:run\"][\"oml:tag\"] = self.tags\n        if (self.fold_evaluations is not None and len(self.fold_evaluations) &gt; 0) or (\n            self.sample_evaluations is not None and len(self.sample_evaluations) &gt; 0\n        ):\n            description[\"oml:run\"][\"oml:output_data\"] = OrderedDict()\n            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"] = []\n        if self.fold_evaluations is not None:\n            for measure in self.fold_evaluations:\n                for repeat in self.fold_evaluations[measure]:\n                    for fold, value in self.fold_evaluations[measure][repeat].items():\n                        current = OrderedDict(\n                            [\n                                (\"@repeat\", str(repeat)),\n                                (\"@fold\", str(fold)),\n                                (\"oml:name\", measure),\n                                (\"oml:value\", str(value)),\n                            ],\n                        )\n                        description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(current)\n        if self.sample_evaluations is not None:\n            for measure in self.sample_evaluations:\n                for repeat in self.sample_evaluations[measure]:\n                    for fold in self.sample_evaluations[measure][repeat]:\n                        for sample, value in self.sample_evaluations[measure][repeat][fold].items():\n                            current = OrderedDict(\n                                [\n                                    (\"@repeat\", str(repeat)),\n                                    (\"@fold\", str(fold)),\n                                    (\"@sample\", str(sample)),\n                                    (\"oml:name\", measure),\n                                    (\"oml:value\", str(value)),\n                                ],\n                            )\n                            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(\n                                current,\n                            )\n        return description\n</code></pre>"},{"location":"reference/#openml.OpenMLRun.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the run, None if not uploaded to the server yet.</p>"},{"location":"reference/#openml.OpenMLRun.predictions","title":"<code>predictions: pd.DataFrame</code>  <code>property</code>","text":"<p>Return a DataFrame with predictions for this run</p>"},{"location":"reference/#openml.OpenMLRun.from_filesystem","title":"<code>from_filesystem(directory, expect_model=True)</code>  <code>classmethod</code>","text":"<p>The inverse of the to_filesystem method. Instantiates an OpenMLRun object based on files stored on the file system.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results are stored</p> required <code>expect_model</code> <code>bool</code> <p>if True, it requires the model pickle to be present, and an error will be thrown if not. Otherwise, the model might or might not be present.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>the re-instantiated run object</p> Source code in <code>openml/runs/run.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n    \"\"\"\n    The inverse of the to_filesystem method. Instantiates an OpenMLRun\n    object based on files stored on the file system.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        are stored\n\n    expect_model : bool\n        if True, it requires the model pickle to be present, and an error\n        will be thrown if not. Otherwise, the model might or might not\n        be present.\n\n    Returns\n    -------\n    run : OpenMLRun\n        the re-instantiated run object\n    \"\"\"\n    # Avoiding cyclic imports\n    import openml.runs.functions\n\n    directory = Path(directory)\n    if not directory.is_dir():\n        raise ValueError(\"Could not find folder\")\n\n    description_path = directory / \"description.xml\"\n    predictions_path = directory / \"predictions.arff\"\n    trace_path = directory / \"trace.arff\"\n    model_path = directory / \"model.pkl\"\n\n    if not description_path.is_file():\n        raise ValueError(\"Could not find description.xml\")\n    if not predictions_path.is_file():\n        raise ValueError(\"Could not find predictions.arff\")\n    if (not model_path.is_file()) and expect_model:\n        raise ValueError(\"Could not find model.pkl\")\n\n    with description_path.open() as fht:\n        xml_string = fht.read()\n    run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n    if run.flow_id is None:\n        flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n        run.flow = flow\n        run.flow_name = flow.name\n\n    with predictions_path.open() as fht:\n        predictions = arff.load(fht)\n        run.data_content = predictions[\"data\"]\n\n    if model_path.is_file():\n        # note that it will load the model if the file exists, even if\n        # expect_model is False\n        with model_path.open(\"rb\") as fhb:\n            run.model = pickle.load(fhb)  # noqa: S301\n\n    if trace_path.is_file():\n        run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n    return run\n</code></pre>"},{"location":"reference/#openml.OpenMLRun.get_metric_fn","title":"<code>get_metric_fn(sklearn_fn, kwargs=None)</code>","text":"<p>Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' or 'truth' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs)</p> <p>Parameters:</p> Name Type Description Default <code>sklearn_fn</code> <code>function</code> <p>a function pointer to a sklearn function that accepts <code>y_true</code>, <code>y_pred</code> and <code>**kwargs</code></p> required <code>kwargs</code> <code>dict</code> <p>kwargs for the function</p> <code>None</code> <p>Returns:</p> Name Type Description <code>scores</code> <code>ndarray of scores of length num_folds * num_repeats</code> <p>metric results</p> Source code in <code>openml/runs/run.py</code> <pre><code>def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n    \"\"\"Calculates metric scores based on predicted values. Assumes the\n    run has been executed locally (and contains run_data). Furthermore,\n    it assumes that the 'correct' or 'truth' attribute is specified in\n    the arff (which is an optional field, but always the case for\n    openml-python runs)\n\n    Parameters\n    ----------\n    sklearn_fn : function\n        a function pointer to a sklearn function that\n        accepts ``y_true``, ``y_pred`` and ``**kwargs``\n    kwargs : dict\n        kwargs for the function\n\n    Returns\n    -------\n    scores : ndarray of scores of length num_folds * num_repeats\n        metric results\n    \"\"\"\n    kwargs = kwargs if kwargs else {}\n    if self.data_content is not None and self.task_id is not None:\n        predictions_arff = self._generate_arff_dict()\n    elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n        predictions_file_url = openml._api_calls._file_id_to_url(\n            self.output_files[\"predictions\"],\n            \"predictions.arff\",\n        )\n        response = openml._api_calls._download_text_file(predictions_file_url)\n        predictions_arff = arff.loads(response)\n        # TODO: make this a stream reader\n    else:\n        raise ValueError(\n            \"Run should have been locally executed or \" \"contain outputfile reference.\",\n        )\n\n    # Need to know more about the task to compute scores correctly\n    task = get_task(self.task_id)\n\n    attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n    if (\n        task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n        and \"correct\" not in attribute_names\n    ):\n        raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n    if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n        raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n    if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n        raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n    def _attribute_list_to_dict(attribute_list):  # type: ignore\n        # convenience function: Creates a mapping to map from the name of\n        # attributes present in the arff prediction file to their index.\n        # This is necessary because the number of classes can be different\n        # for different tasks.\n        res = OrderedDict()\n        for idx in range(len(attribute_list)):\n            res[attribute_list[idx][0]] = idx\n        return res\n\n    attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n    repeat_idx = attribute_dict[\"repeat\"]\n    fold_idx = attribute_dict[\"fold\"]\n    predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n    if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n        correct_idx = attribute_dict[\"correct\"]\n    elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n        correct_idx = attribute_dict[\"truth\"]\n    has_samples = False\n    if \"sample\" in attribute_dict:\n        sample_idx = attribute_dict[\"sample\"]\n        has_samples = True\n\n    if (\n        predictions_arff[\"attributes\"][predicted_idx][1]\n        != predictions_arff[\"attributes\"][correct_idx][1]\n    ):\n        pred = predictions_arff[\"attributes\"][predicted_idx][1]\n        corr = predictions_arff[\"attributes\"][correct_idx][1]\n        raise ValueError(\n            \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n        )\n\n    # TODO: these could be cached\n    values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n    values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n    for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n        rep = line[repeat_idx]\n        fold = line[fold_idx]\n        samp = line[sample_idx] if has_samples else 0\n\n        if task.task_type_id in [\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.LEARNING_CURVE,\n        ]:\n            prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                line[predicted_idx],\n            )\n            correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            prediction = line[predicted_idx]\n            correct = line[correct_idx]\n        if rep not in values_predict:\n            values_predict[rep] = OrderedDict()\n            values_correct[rep] = OrderedDict()\n        if fold not in values_predict[rep]:\n            values_predict[rep][fold] = OrderedDict()\n            values_correct[rep][fold] = OrderedDict()\n        if samp not in values_predict[rep][fold]:\n            values_predict[rep][fold][samp] = []\n            values_correct[rep][fold][samp] = []\n\n        values_predict[rep][fold][samp].append(prediction)\n        values_correct[rep][fold][samp].append(correct)\n\n    scores = []\n    for rep in values_predict:\n        for fold in values_predict[rep]:\n            last_sample = len(values_predict[rep][fold]) - 1\n            y_pred = values_predict[rep][fold][last_sample]\n            y_true = values_correct[rep][fold][last_sample]\n            scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n    return np.array(scores)\n</code></pre>"},{"location":"reference/#openml.OpenMLRun.to_filesystem","title":"<code>to_filesystem(directory, store_model=True)</code>","text":"<p>The inverse of the from_filesystem method. Serializes a run on the filesystem, to be uploaded later.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results will be stored. Should be empty</p> required <code>store_model</code> <code>(bool, optional(default=True))</code> <p>if True, a model will be pickled as well. As this is the most storage expensive part, it is often desirable to not store the model.</p> <code>True</code> Source code in <code>openml/runs/run.py</code> <pre><code>def to_filesystem(\n    self,\n    directory: str | Path,\n    store_model: bool = True,  # noqa: FBT001, FBT002\n) -&gt; None:\n    \"\"\"\n    The inverse of the from_filesystem method. Serializes a run\n    on the filesystem, to be uploaded later.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        will be stored. Should be empty\n\n    store_model : bool, optional (default=True)\n        if True, a model will be pickled as well. As this is the most\n        storage expensive part, it is often desirable to not store the\n        model.\n    \"\"\"\n    if self.data_content is None or self.model is None:\n        raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n    directory = Path(directory)\n    directory.mkdir(exist_ok=True, parents=True)\n\n    if any(directory.iterdir()):\n        raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n    run_xml = self._to_xml()\n    predictions_arff = arff.dumps(self._generate_arff_dict())\n\n    # It seems like typing does not allow to define the same variable multiple times\n    with (directory / \"description.xml\").open(\"w\") as fh:\n        fh.write(run_xml)\n    with (directory / \"predictions.arff\").open(\"w\") as fh:\n        fh.write(predictions_arff)\n    if store_model:\n        with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n            pickle.dump(self.model, fh_b)\n\n    if self.flow_id is None and self.flow is not None:\n        self.flow.to_filesystem(directory)\n\n    if self.trace is not None:\n        self.trace._to_filesystem(directory)\n</code></pre>"},{"location":"reference/#openml.OpenMLSetup","title":"<code>OpenMLSetup</code>","text":"<p>Setup object (a.k.a. Configuration).</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The OpenML setup id</p> required <code>flow_id</code> <code>int</code> <p>The flow that it is build upon</p> required <code>parameters</code> <code>dict</code> <p>The setting of the parameters</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLSetup:\n    \"\"\"Setup object (a.k.a. Configuration).\n\n    Parameters\n    ----------\n    setup_id : int\n        The OpenML setup id\n    flow_id : int\n        The flow that it is build upon\n    parameters : dict\n        The setting of the parameters\n    \"\"\"\n\n    def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None):\n        if not isinstance(setup_id, int):\n            raise ValueError(\"setup id should be int\")\n\n        if not isinstance(flow_id, int):\n            raise ValueError(\"flow id should be int\")\n\n        if parameters is not None and not isinstance(parameters, dict):\n            raise ValueError(\"parameters should be dict\")\n\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.parameters = parameters\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Setup\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Setup ID\": self.setup_id,\n            \"Flow ID\": self.flow_id,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"# of Parameters\": (\n                len(self.parameters) if self.parameters is not None else float(\"nan\")\n            ),\n        }\n\n        # determines the order in which the information will be printed\n        order = [\"Setup ID\", \"Flow ID\", \"Flow URL\", \"# of Parameters\"]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/#openml.OpenMLSplit","title":"<code>OpenMLSplit</code>","text":"<p>OpenML Split object.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>int or str</code> required <code>description</code> <code>str</code> required <code>split</code> <code>dict</code> required Source code in <code>openml/tasks/split.py</code> <pre><code>class OpenMLSplit:\n    \"\"\"OpenML Split object.\n\n    Parameters\n    ----------\n    name : int or str\n    description : str\n    split : dict\n    \"\"\"\n\n    def __init__(\n        self,\n        name: int | str,\n        description: str,\n        split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]],\n    ):\n        self.description = description\n        self.name = name\n        self.split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]] = {}\n\n        # Add splits according to repetition\n        for repetition in split:\n            _rep = int(repetition)\n            self.split[_rep] = OrderedDict()\n            for fold in split[_rep]:\n                self.split[_rep][fold] = OrderedDict()\n                for sample in split[_rep][fold]:\n                    self.split[_rep][fold][sample] = split[_rep][fold][sample]\n\n        self.repeats = len(self.split)\n\n        # TODO(eddiebergman): Better error message\n        if any(len(self.split[0]) != len(self.split[i]) for i in range(self.repeats)):\n            raise ValueError(\"\")\n\n        self.folds = len(self.split[0])\n        self.samples = len(self.split[0][0])\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if (\n            (not isinstance(self, type(other)))\n            or self.name != other.name\n            or self.description != other.description\n            or self.split.keys() != other.split.keys()\n            or any(\n                self.split[repetition].keys() != other.split[repetition].keys()\n                for repetition in self.split\n            )\n        ):\n            return False\n\n        samples = [\n            (repetition, fold, sample)\n            for repetition in self.split\n            for fold in self.split[repetition]\n            for sample in self.split[repetition][fold]\n        ]\n\n        for repetition, fold, sample in samples:\n            self_train, self_test = self.split[repetition][fold][sample]\n            other_train, other_test = other.split[repetition][fold][sample]\n            if not (np.all(self_train == other_train) and np.all(self_test == other_test)):\n                return False\n        return True\n\n    @classmethod\n    def _from_arff_file(cls, filename: Path) -&gt; OpenMLSplit:  # noqa: C901, PLR0912\n        repetitions = None\n        name = None\n\n        pkl_filename = filename.with_suffix(\".pkl.py3\")\n\n        if pkl_filename.exists():\n            with pkl_filename.open(\"rb\") as fh:\n                # TODO(eddiebergman): Would be good to figure out what _split is and assert it is\n                _split = pickle.load(fh)  # noqa: S301\n            repetitions = _split[\"repetitions\"]\n            name = _split[\"name\"]\n\n        # Cache miss\n        if repetitions is None:\n            # Faster than liac-arff and sufficient in this situation!\n            if not filename.exists():\n                raise FileNotFoundError(f\"Split arff {filename} does not exist!\")\n\n            file_data = arff.load(filename.open(\"r\"), return_type=arff.DENSE_GEN)\n            splits = file_data[\"data\"]\n            name = file_data[\"relation\"]\n            attrnames = [attr[0] for attr in file_data[\"attributes\"]]\n\n            repetitions = OrderedDict()\n\n            type_idx = attrnames.index(\"type\")\n            rowid_idx = attrnames.index(\"rowid\")\n            repeat_idx = attrnames.index(\"repeat\")\n            fold_idx = attrnames.index(\"fold\")\n            sample_idx = attrnames.index(\"sample\") if \"sample\" in attrnames else None\n\n            for line in splits:\n                # A line looks like type, rowid, repeat, fold\n                repetition = int(line[repeat_idx])\n                fold = int(line[fold_idx])\n                sample = 0\n                if sample_idx is not None:\n                    sample = int(line[sample_idx])\n\n                if repetition not in repetitions:\n                    repetitions[repetition] = OrderedDict()\n                if fold not in repetitions[repetition]:\n                    repetitions[repetition][fold] = OrderedDict()\n                if sample not in repetitions[repetition][fold]:\n                    repetitions[repetition][fold][sample] = ([], [])\n                split = repetitions[repetition][fold][sample]\n\n                type_ = line[type_idx]\n                if type_ == \"TRAIN\":\n                    split[0].append(line[rowid_idx])\n                elif type_ == \"TEST\":\n                    split[1].append(line[rowid_idx])\n                else:\n                    raise ValueError(type_)\n\n            for repetition in repetitions:\n                for fold in repetitions[repetition]:\n                    for sample in repetitions[repetition][fold]:\n                        repetitions[repetition][fold][sample] = Split(\n                            np.array(repetitions[repetition][fold][sample][0], dtype=np.int32),\n                            np.array(repetitions[repetition][fold][sample][1], dtype=np.int32),\n                        )\n\n            with pkl_filename.open(\"wb\") as fh:\n                pickle.dump({\"name\": name, \"repetitions\": repetitions}, fh, protocol=2)\n\n        assert name is not None\n        return cls(name, \"\", repetitions)\n\n    def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n        Parameters\n        ----------\n        repeat : int\n            Index of the repeat to retrieve.\n        fold : int\n            Index of the fold to retrieve.\n        sample : int\n            Index of the sample to retrieve.\n\n        Returns\n        -------\n        numpy.ndarray\n            The data split for the specified repeat, fold, and sample.\n\n        Raises\n        ------\n        ValueError\n            If the specified repeat, fold, or sample is not known.\n        \"\"\"\n        if repeat not in self.split:\n            raise ValueError(\"Repeat %s not known\" % str(repeat))\n        if fold not in self.split[repeat]:\n            raise ValueError(\"Fold %s not known\" % str(fold))\n        if sample not in self.split[repeat][fold]:\n            raise ValueError(\"Sample %s not known\" % str(sample))\n        return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/#openml.OpenMLSplit.get","title":"<code>get(repeat=0, fold=0, sample=0)</code>","text":"<p>Returns the specified data split from the CrossValidationSplit object.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>Index of the repeat to retrieve.</p> <code>0</code> <code>fold</code> <code>int</code> <p>Index of the fold to retrieve.</p> <code>0</code> <code>sample</code> <code>int</code> <p>Index of the sample to retrieve.</p> <code>0</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>The data split for the specified repeat, fold, and sample.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the specified repeat, fold, or sample is not known.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n    Parameters\n    ----------\n    repeat : int\n        Index of the repeat to retrieve.\n    fold : int\n        Index of the fold to retrieve.\n    sample : int\n        Index of the sample to retrieve.\n\n    Returns\n    -------\n    numpy.ndarray\n        The data split for the specified repeat, fold, and sample.\n\n    Raises\n    ------\n    ValueError\n        If the specified repeat, fold, or sample is not known.\n    \"\"\"\n    if repeat not in self.split:\n        raise ValueError(\"Repeat %s not known\" % str(repeat))\n    if fold not in self.split[repeat]:\n        raise ValueError(\"Fold %s not known\" % str(fold))\n    if sample not in self.split[repeat][fold]:\n        raise ValueError(\"Sample %s not known\" % str(sample))\n    return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/#openml.OpenMLStudy","title":"<code>OpenMLStudy</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLStudy represents the OpenML concept of a study (a collection of runs).</p> <p>It contains the following information: name, id, description, creation date, creator id and a list of run ids.</p> <p>According to this list of run ids, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLStudy(BaseStudy):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study (a collection of runs).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and a list of run ids.\n\n    According to this list of run ids, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        super().__init__(\n            study_id=study_id,\n            alias=alias,\n            main_entity_type=\"run\",\n            benchmark_suite=benchmark_suite,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=flows,\n            runs=runs,\n            setups=setups,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLSupervisedTask","title":"<code>OpenMLSupervisedTask</code>","text":"<p>               Bases: <code>OpenMLTask</code>, <code>ABC</code></p> <p>OpenML Supervised Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature (the class variable).</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure for the task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure for the task.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the task.</p> <code>None</code> <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLSupervisedTask(OpenMLTask, ABC):\n    \"\"\"OpenML Supervised Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the task type.\n    task_type : str\n        Name of the task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the task.\n    target_name : str\n        Name of the target feature (the class variable).\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure for the task.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure for the task.\n    data_splits_url : str, default=None\n        URL of the data splits for the task.\n    task_id: Union[int, None]\n        Refers to the unique identifier of task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | scipy.sparse.spmatrix,\n        np.ndarray | None,\n    ]:\n        ...\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\"]\n    ) -&gt; tuple[\n        pd.DataFrame,\n        pd.Series | pd.DataFrame | None,\n    ]:\n        ...\n\n    # TODO(eddiebergman): Do all OpenMLSupervisedTask have a `y`?\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n        np.ndarray | pd.Series | pd.DataFrame | None,\n    ]:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        dataset = self.get_dataset()\n        if self.task_type_id not in (\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.SUPERVISED_REGRESSION,\n            TaskType.LEARNING_CURVE,\n        ):\n            raise NotImplementedError(self.task_type)\n\n        X, y, _, _ = dataset.get_data(\n            dataset_format=dataset_format,\n            target=self.target_name,\n        )\n        return X, y\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        task_container = super()._to_dict()\n        oml_input = task_container[\"oml:task_inputs\"][\"oml:input\"]  # type: ignore\n        assert isinstance(oml_input, list)\n\n        oml_input.append({\"@name\": \"target_feature\", \"#text\": self.target_name})\n        return task_container\n\n    @property\n    def estimation_parameters(self) -&gt; dict[str, str] | None:\n        \"\"\"Return the estimation parameters for the task.\"\"\"\n        warnings.warn(\n            \"The estimation_parameters attribute will be \"\n            \"deprecated in the future, please use \"\n            \"estimation_procedure['parameters'] instead\",\n            PendingDeprecationWarning,\n            stacklevel=2,\n        )\n        return self.estimation_procedure[\"parameters\"]\n\n    @estimation_parameters.setter\n    def estimation_parameters(self, est_parameters: dict[str, str] | None) -&gt; None:\n        self.estimation_procedure[\"parameters\"] = est_parameters\n</code></pre>"},{"location":"reference/#openml.OpenMLSupervisedTask.estimation_parameters","title":"<code>estimation_parameters: dict[str, str] | None</code>  <code>property</code> <code>writable</code>","text":"<p>Return the estimation parameters for the task.</p>"},{"location":"reference/#openml.OpenMLSupervisedTask.get_X_and_y","title":"<code>get_X_and_y(dataset_format='array')</code>","text":"<pre><code>get_X_and_y(dataset_format: Literal['array'] = 'array') -&gt; tuple[np.ndarray | scipy.sparse.spmatrix, np.ndarray | None]\n</code></pre><pre><code>get_X_and_y(dataset_format: Literal['dataframe']) -&gt; tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X_and_y(\n    self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n    np.ndarray | pd.Series | pd.DataFrame | None,\n]:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    dataset = self.get_dataset()\n    if self.task_type_id not in (\n        TaskType.SUPERVISED_CLASSIFICATION,\n        TaskType.SUPERVISED_REGRESSION,\n        TaskType.LEARNING_CURVE,\n    ):\n        raise NotImplementedError(self.task_type)\n\n    X, y, _, _ = dataset.get_data(\n        dataset_format=dataset_format,\n        target=self.target_name,\n    )\n    return X, y\n</code></pre>"},{"location":"reference/#openml.OpenMLTask","title":"<code>OpenMLTask</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Task object.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of OpenML task.</p> required <code>task_type_id</code> <code>TaskType</code> <p>Refers to the type of OpenML task.</p> required <code>task_type</code> <code>str</code> <p>Refers to the OpenML task.</p> required <code>data_set_id</code> <code>int</code> <p>Refers to the data.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>Refers to the type of estimates used.</p> <code>1</code> <code>estimation_procedure_type</code> <code>str | None</code> <p>Refers to the type of estimation procedure used for the OpenML task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict[str, str] | None</code> <p>Estimation parameters used for the OpenML task.</p> <code>None</code> <code>evaluation_measure</code> <code>str | None</code> <p>Refers to the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str | None</code> <p>Refers to the URL of the data splits used for the OpenML task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLTask(OpenMLBase):\n    \"\"\"OpenML Task object.\n\n    Parameters\n    ----------\n    task_id: Union[int, None]\n        Refers to the unique identifier of OpenML task.\n    task_type_id: TaskType\n        Refers to the type of OpenML task.\n    task_type: str\n        Refers to the OpenML task.\n    data_set_id: int\n        Refers to the data.\n    estimation_procedure_id: int\n        Refers to the type of estimates used.\n    estimation_procedure_type: str, default=None\n        Refers to the type of estimation procedure used for the OpenML task.\n    estimation_parameters: [Dict[str, str]], default=None\n        Estimation parameters used for the OpenML task.\n    evaluation_measure: str, default=None\n        Refers to the evaluation measure.\n    data_splits_url: str, default=None\n        Refers to the URL of the data splits used for the OpenML task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int | None,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n    ):\n        self.task_id = int(task_id) if task_id is not None else None\n        self.task_type_id = task_type_id\n        self.task_type = task_type\n        self.dataset_id = int(data_set_id)\n        self.evaluation_measure = evaluation_measure\n        self.estimation_procedure: _EstimationProcedure = {\n            \"type\": estimation_procedure_type,\n            \"parameters\": estimation_parameters,\n            \"data_splits_url\": data_splits_url,\n        }\n        self.estimation_procedure_id = estimation_procedure_id\n        self.split: OpenMLSplit | None = None\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"t\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the OpenML ID of this task.\"\"\"\n        return self.task_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        base_server_url = openml.config.get_server_base_url()\n        fields: dict[str, Any] = {\n            \"Task Type Description\": f\"{base_server_url}/tt/{self.task_type_id}\"\n        }\n        if self.task_id is not None:\n            fields[\"Task ID\"] = self.task_id\n            fields[\"Task URL\"] = self.openml_url\n        if self.evaluation_measure is not None:\n            fields[\"Evaluation Measure\"] = self.evaluation_measure\n        if self.estimation_procedure is not None:\n            fields[\"Estimation Procedure\"] = self.estimation_procedure[\"type\"]\n\n        # TODO(eddiebergman): Subclasses could advertise/provide this, instead of having to\n        # have the base class know about it's subclasses.\n        target_name = getattr(self, \"target_name\", None)\n        if target_name is not None:\n            fields[\"Target Feature\"] = target_name\n\n            class_labels = getattr(self, \"class_labels\", None)\n            if class_labels is not None:\n                fields[\"# of Classes\"] = len(class_labels)\n\n            if hasattr(self, \"cost_matrix\"):\n                fields[\"Cost Matrix\"] = \"Available\"\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Task Type Description\",\n            \"Task ID\",\n            \"Task URL\",\n            \"Estimation Procedure\",\n            \"Evaluation Measure\",\n            \"Target Feature\",\n            \"# of Classes\",\n            \"Cost Matrix\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def get_dataset(self) -&gt; datasets.OpenMLDataset:\n        \"\"\"Download dataset associated with task.\"\"\"\n        return datasets.get_dataset(self.dataset_id)\n\n    def get_train_test_split_indices(\n        self,\n        fold: int = 0,\n        repeat: int = 0,\n        sample: int = 0,\n    ) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n        # Replace with retrieve from cache\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.get(repeat=repeat, fold=fold, sample=sample)\n\n    def _download_split(self, cache_file: Path) -&gt; None:\n        # TODO(eddiebergman): Not sure about this try to read and error approach\n        try:\n            with cache_file.open(encoding=\"utf8\"):\n                pass\n        except OSError:\n            split_url = self.estimation_procedure[\"data_splits_url\"]\n            openml._api_calls._download_text_file(\n                source=str(split_url),\n                output_path=str(cache_file),\n            )\n\n    def download_split(self) -&gt; OpenMLSplit:\n        \"\"\"Download the OpenML split for a given task.\"\"\"\n        # TODO(eddiebergman): Can this every be `None`?\n        assert self.task_id is not None\n        cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n        cached_split_file = cache_dir / \"datasplits.arff\"\n\n        try:\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n        except OSError:\n            # Next, download and cache the associated split file\n            self._download_split(cached_split_file)\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n\n        return split\n\n    def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n        \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.repeats, self.split.folds, self.split.samples\n\n    # TODO(eddiebergman): Really need some better typing on all this\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        \"\"\"Creates a dictionary representation of self in a string format (for XML parsing).\"\"\"\n        oml_input = [\n            {\"@name\": \"source_data\", \"#text\": str(self.dataset_id)},\n            {\"@name\": \"estimation_procedure\", \"#text\": str(self.estimation_procedure_id)},\n        ]\n        if self.evaluation_measure is not None:  #\n            oml_input.append({\"@name\": \"evaluation_measures\", \"#text\": self.evaluation_measure})\n\n        return {\n            \"oml:task_inputs\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                \"oml:task_type_id\": self.task_type_id.value,  # This is an int from the enum?\n                \"oml:input\": oml_input,\n            }\n        }\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.task_id = int(xml_response[\"oml:upload_task\"][\"oml:id\"])\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the OpenML ID of this task.</p>"},{"location":"reference/#openml.OpenMLTask.download_split","title":"<code>download_split()</code>","text":"<p>Download the OpenML split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def download_split(self) -&gt; OpenMLSplit:\n    \"\"\"Download the OpenML split for a given task.\"\"\"\n    # TODO(eddiebergman): Can this every be `None`?\n    assert self.task_id is not None\n    cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n    cached_split_file = cache_dir / \"datasplits.arff\"\n\n    try:\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n    except OSError:\n        # Next, download and cache the associated split file\n        self._download_split(cached_split_file)\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n\n    return split\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.get_dataset","title":"<code>get_dataset()</code>","text":"<p>Download dataset associated with task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_dataset(self) -&gt; datasets.OpenMLDataset:\n    \"\"\"Download dataset associated with task.\"\"\"\n    return datasets.get_dataset(self.dataset_id)\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.get_split_dimensions","title":"<code>get_split_dimensions()</code>","text":"<p>Get the (repeats, folds, samples) of the split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n    \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.repeats, self.split.folds, self.split.samples\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.get_train_test_split_indices","title":"<code>get_train_test_split_indices(fold=0, repeat=0, sample=0)</code>","text":"<p>Get the indices of the train and test splits for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_train_test_split_indices(\n    self,\n    fold: int = 0,\n    repeat: int = 0,\n    sample: int = 0,\n) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n    # Replace with retrieve from cache\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.get(repeat=repeat, fold=fold, sample=sample)\n</code></pre>"},{"location":"reference/#openml.populate_cache","title":"<code>populate_cache(task_ids=None, dataset_ids=None, flow_ids=None, run_ids=None)</code>","text":"<p>Populate a cache for offline and parallel usage of the OpenML connector.</p> <p>Parameters:</p> Name Type Description Default <code>task_ids</code> <code>iterable</code> <code>None</code> <code>dataset_ids</code> <code>iterable</code> <code>None</code> <code>flow_ids</code> <code>iterable</code> <code>None</code> <code>run_ids</code> <code>iterable</code> <code>None</code> <p>Returns:</p> Type Description <code>None</code> Source code in <code>openml/__init__.py</code> <pre><code>def populate_cache(\n    task_ids: list[int] | None = None,\n    dataset_ids: list[int | str] | None = None,\n    flow_ids: list[int] | None = None,\n    run_ids: list[int] | None = None,\n) -&gt; None:\n    \"\"\"\n    Populate a cache for offline and parallel usage of the OpenML connector.\n\n    Parameters\n    ----------\n    task_ids : iterable\n\n    dataset_ids : iterable\n\n    flow_ids : iterable\n\n    run_ids : iterable\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if task_ids is not None:\n        for task_id in task_ids:\n            tasks.functions.get_task(task_id)\n\n    if dataset_ids is not None:\n        for dataset_id in dataset_ids:\n            datasets.functions.get_dataset(dataset_id)\n\n    if flow_ids is not None:\n        for flow_id in flow_ids:\n            flows.functions.get_flow(flow_id)\n\n    if run_ids is not None:\n        for run_id in run_ids:\n            runs.functions.get_run(run_id)\n</code></pre>"},{"location":"reference/SUMMARY/","title":"SUMMARY","text":"<ul> <li>base</li> <li>cli</li> <li>config</li> <li>datasets<ul> <li>data_feature</li> <li>dataset</li> <li>functions</li> </ul> </li> <li>evaluations<ul> <li>evaluation</li> <li>functions</li> </ul> </li> <li>exceptions</li> <li>extensions<ul> <li>extension_interface</li> <li>functions</li> <li>sklearn<ul> <li>extension</li> </ul> </li> </ul> </li> <li>flows<ul> <li>flow</li> <li>functions</li> </ul> </li> <li>runs<ul> <li>functions</li> <li>run</li> <li>trace</li> </ul> </li> <li>setups<ul> <li>functions</li> <li>setup</li> </ul> </li> <li>study<ul> <li>functions</li> <li>study</li> </ul> </li> <li>tasks<ul> <li>functions</li> <li>split</li> <li>task</li> </ul> </li> <li>testing</li> <li>utils</li> </ul>"},{"location":"reference/__version__/","title":"version","text":"<p>Version information.</p>"},{"location":"reference/_api_calls/","title":"api calls","text":""},{"location":"reference/_api_calls/#openml._api_calls.resolve_env_proxies","title":"<code>resolve_env_proxies(url)</code>","text":"<p>Attempt to find a suitable proxy for this url.</p> <p>Relies on <code>requests</code> internals to remain consistent. To disable this from the environment, please set the enviornment varialbe <code>no_proxy=\"*\"</code>.</p> <p>Parameters:</p> Name Type Description Default <code>url</code> <code>str</code> <p>The url endpoint</p> required <p>Returns:</p> Type Description <code>Optional[str]</code> <p>The proxy url if found, else None</p> Source code in <code>openml/_api_calls.py</code> <pre><code>def resolve_env_proxies(url: str) -&gt; str | None:\n    \"\"\"Attempt to find a suitable proxy for this url.\n\n    Relies on ``requests`` internals to remain consistent. To disable this from the\n    environment, please set the enviornment varialbe ``no_proxy=\"*\"``.\n\n    Parameters\n    ----------\n    url : str\n        The url endpoint\n\n    Returns\n    -------\n    Optional[str]\n        The proxy url if found, else None\n    \"\"\"\n    resolved_proxies = requests.utils.get_environ_proxies(url)\n    return requests.utils.select_proxy(url, resolved_proxies)  # type: ignore\n</code></pre>"},{"location":"reference/base/","title":"base","text":""},{"location":"reference/base/#openml.base.OpenMLBase","title":"<code>OpenMLBase</code>","text":"<p>               Bases: <code>ABC</code></p> <p>Base object for functionality that is shared across entities.</p> Source code in <code>openml/base.py</code> <pre><code>class OpenMLBase(ABC):\n    \"\"\"Base object for functionality that is shared across entities.\"\"\"\n\n    def __repr__(self) -&gt; str:\n        body_fields = self._get_repr_body_fields()\n        return self._apply_repr_template(body_fields)\n\n    @property\n    @abstractmethod\n    def id(self) -&gt; int | None:\n        \"\"\"The id of the entity, it is unique for its entity type.\"\"\"\n\n    @property\n    def openml_url(self) -&gt; str | None:\n        \"\"\"The URL of the object on the server, if it was uploaded, else None.\"\"\"\n        if self.id is None:\n            return None\n        return self.__class__.url_for_id(self.id)\n\n    @classmethod\n    def url_for_id(cls, id_: int) -&gt; str:\n        \"\"\"Return the OpenML URL for the object of the class entity with the given id.\"\"\"\n        # Sample url for a flow: openml.org/f/123\n        return f\"{openml.config.get_server_base_url()}/{cls._entity_letter()}/{id_}\"\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        \"\"\"Return the letter which represents the entity type in urls, e.g. 'f' for flow.\"\"\"\n        # We take advantage of the class naming convention (OpenMLX),\n        # which holds for all entities except studies and tasks, which overwrite this method.\n        return cls.__name__.lower()[len(\"OpenML\") :][0]\n\n    # TODO(eddiebergman): This would be much cleaner as an iterator...\n    @abstractmethod\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str] | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\n\n        Returns\n        -------\n        body_fields : List[Tuple[str, Union[str, int, List[str]]]]\n            A list of (name, value) pairs to display in the body of the __repr__.\n            E.g.: [('metric', 'accuracy'), ('dataset', 'iris')]\n            If value is a List of str, then each item of the list will appear in a separate row.\n        \"\"\"\n        # Should be implemented in the base class.\n\n    def _apply_repr_template(\n        self,\n        body_fields: Iterable[tuple[str, str | int | list[str] | None]],\n    ) -&gt; str:\n        \"\"\"Generates the header and formats the body for string representation of the object.\n\n        Parameters\n        ----------\n        body_fields: List[Tuple[str, str]]\n           A list of (name, value) pairs to display in the body of the __repr__.\n        \"\"\"\n        # We add spaces between capitals, e.g. ClassificationTask -&gt; Classification Task\n        name_with_spaces = re.sub(\n            r\"(\\w)([A-Z])\",\n            r\"\\1 \\2\",\n            self.__class__.__name__[len(\"OpenML\") :],\n        )\n        header_text = f\"OpenML {name_with_spaces}\"\n        header = \"{}\\n{}\\n\".format(header_text, \"=\" * len(header_text))\n\n        _body_fields: list[tuple[str, str | int | list[str]]] = [\n            (k, \"None\" if v is None else v) for k, v in body_fields\n        ]\n        longest_field_name_length = max(len(name) for name, _ in _body_fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _body_fields)\n        return header + body\n\n    @abstractmethod\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\n\n        The return value will be used to create the upload xml file.\n        The xml file must have the tags in exactly the order of the object's xsd.\n        (see https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/).\n\n        Returns\n        -------\n            Thing represented as dict.\n        \"\"\"\n        # Should be implemented in the base class.\n\n    def _to_xml(self) -&gt; str:\n        \"\"\"Generate xml representation of self for upload to server.\"\"\"\n        dict_representation = self._to_dict()\n        xml_representation = xmltodict.unparse(dict_representation, pretty=True)\n\n        # A task may not be uploaded with the xml encoding specification:\n        # &lt;?xml version=\"1.0\" encoding=\"utf-8\"?&gt;\n        _encoding_specification, xml_body = xml_representation.split(\"\\n\", 1)\n        return str(xml_body)\n\n    def _get_file_elements(self) -&gt; openml._api_calls.FILE_ELEMENTS_TYPE:\n        \"\"\"Get file_elements to upload to the server, called during Publish.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        return {}\n\n    @abstractmethod\n    def _parse_publish_response(self, xml_response: dict[str, str]) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n\n    def publish(self) -&gt; OpenMLBase:\n        \"\"\"Publish the object on the OpenML server.\"\"\"\n        file_elements = self._get_file_elements()\n\n        if \"description\" not in file_elements:\n            file_elements[\"description\"] = self._to_xml()\n\n        call = f\"{_get_rest_api_type_alias(self)}/\"\n        response_text = openml._api_calls._perform_api_call(\n            call,\n            \"post\",\n            file_elements=file_elements,\n        )\n        xml_response = xmltodict.parse(response_text)\n\n        self._parse_publish_response(xml_response)\n        return self\n\n    def open_in_browser(self) -&gt; None:\n        \"\"\"Opens the OpenML web page corresponding to this object in your default browser.\"\"\"\n        if self.openml_url is None:\n            raise ValueError(\n                \"Cannot open element on OpenML.org when attribute `openml_url` is `None`\",\n            )\n\n        webbrowser.open(self.openml_url)\n\n    def push_tag(self, tag: str) -&gt; None:\n        \"\"\"Annotates this entity with a tag on the server.\n\n        Parameters\n        ----------\n        tag : str\n            Tag to attach to the flow.\n        \"\"\"\n        _tag_openml_base(self, tag)\n\n    def remove_tag(self, tag: str) -&gt; None:\n        \"\"\"Removes a tag from this entity on the server.\n\n        Parameters\n        ----------\n        tag : str\n            Tag to attach to the flow.\n        \"\"\"\n        _tag_openml_base(self, tag, untag=True)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.id","title":"<code>id: int | None</code>  <code>abstractmethod</code> <code>property</code>","text":"<p>The id of the entity, it is unique for its entity type.</p>"},{"location":"reference/base/#openml.base.OpenMLBase.openml_url","title":"<code>openml_url: str | None</code>  <code>property</code>","text":"<p>The URL of the object on the server, if it was uploaded, else None.</p>"},{"location":"reference/base/#openml.base.OpenMLBase.open_in_browser","title":"<code>open_in_browser()</code>","text":"<p>Opens the OpenML web page corresponding to this object in your default browser.</p> Source code in <code>openml/base.py</code> <pre><code>def open_in_browser(self) -&gt; None:\n    \"\"\"Opens the OpenML web page corresponding to this object in your default browser.\"\"\"\n    if self.openml_url is None:\n        raise ValueError(\n            \"Cannot open element on OpenML.org when attribute `openml_url` is `None`\",\n        )\n\n    webbrowser.open(self.openml_url)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.publish","title":"<code>publish()</code>","text":"<p>Publish the object on the OpenML server.</p> Source code in <code>openml/base.py</code> <pre><code>def publish(self) -&gt; OpenMLBase:\n    \"\"\"Publish the object on the OpenML server.\"\"\"\n    file_elements = self._get_file_elements()\n\n    if \"description\" not in file_elements:\n        file_elements[\"description\"] = self._to_xml()\n\n    call = f\"{_get_rest_api_type_alias(self)}/\"\n    response_text = openml._api_calls._perform_api_call(\n        call,\n        \"post\",\n        file_elements=file_elements,\n    )\n    xml_response = xmltodict.parse(response_text)\n\n    self._parse_publish_response(xml_response)\n    return self\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.push_tag","title":"<code>push_tag(tag)</code>","text":"<p>Annotates this entity with a tag on the server.</p> <p>Parameters:</p> Name Type Description Default <code>tag</code> <code>str</code> <p>Tag to attach to the flow.</p> required Source code in <code>openml/base.py</code> <pre><code>def push_tag(self, tag: str) -&gt; None:\n    \"\"\"Annotates this entity with a tag on the server.\n\n    Parameters\n    ----------\n    tag : str\n        Tag to attach to the flow.\n    \"\"\"\n    _tag_openml_base(self, tag)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.remove_tag","title":"<code>remove_tag(tag)</code>","text":"<p>Removes a tag from this entity on the server.</p> <p>Parameters:</p> Name Type Description Default <code>tag</code> <code>str</code> <p>Tag to attach to the flow.</p> required Source code in <code>openml/base.py</code> <pre><code>def remove_tag(self, tag: str) -&gt; None:\n    \"\"\"Removes a tag from this entity on the server.\n\n    Parameters\n    ----------\n    tag : str\n        Tag to attach to the flow.\n    \"\"\"\n    _tag_openml_base(self, tag, untag=True)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.url_for_id","title":"<code>url_for_id(id_)</code>  <code>classmethod</code>","text":"<p>Return the OpenML URL for the object of the class entity with the given id.</p> Source code in <code>openml/base.py</code> <pre><code>@classmethod\ndef url_for_id(cls, id_: int) -&gt; str:\n    \"\"\"Return the OpenML URL for the object of the class entity with the given id.\"\"\"\n    # Sample url for a flow: openml.org/f/123\n    return f\"{openml.config.get_server_base_url()}/{cls._entity_letter()}/{id_}\"\n</code></pre>"},{"location":"reference/cli/","title":"cli","text":"<p>\"Command Line Interface for <code>openml</code> to configure its settings.</p>"},{"location":"reference/cli/#openml.cli.configure","title":"<code>configure(args)</code>","text":"<p>Calls the right submenu(s) to edit <code>args.field</code> in the configuration file.</p> Source code in <code>openml/cli.py</code> <pre><code>def configure(args: argparse.Namespace) -&gt; None:\n    \"\"\"Calls the right submenu(s) to edit `args.field` in the configuration file.\"\"\"\n    set_functions = {\n        \"apikey\": configure_apikey,\n        \"server\": configure_server,\n        \"cachedir\": configure_cachedir,\n        \"retry_policy\": configure_retry_policy,\n        \"connection_n_retries\": configure_connection_n_retries,\n        \"avoid_duplicate_runs\": configure_avoid_duplicate_runs,\n        \"verbosity\": configure_verbosity,\n    }\n\n    def not_supported_yet(_: str) -&gt; None:\n        print(f\"Setting '{args.field}' is not supported yet.\")\n\n    if args.field not in [\"all\", \"none\"]:\n        set_functions.get(args.field, not_supported_yet)(args.value)\n    else:\n        if args.value is not None:\n            print(f\"Can not set value ('{args.value}') when field is specified as '{args.field}'.\")\n            sys.exit()\n        print_configuration()\n\n    if args.field == \"all\":\n        for set_field_function in set_functions.values():\n            set_field_function(args.value)\n</code></pre>"},{"location":"reference/cli/#openml.cli.configure_field","title":"<code>configure_field(field, value, check_with_message, intro_message, input_message, sanitize=None)</code>","text":"<p>Configure <code>field</code> with <code>value</code>. If <code>value</code> is None ask the user for input.</p> <p><code>value</code> and user input are first corrected/auto-completed with <code>convert_value</code> if provided, then validated with <code>check_with_message</code> function. If the user input a wrong value in interactive mode, the user gets to input a new value. The new valid value is saved in the openml configuration file. In case an invalid <code>value</code> is supplied directly (non-interactive), no changes are made.</p> <p>Parameters:</p> Name Type Description Default <code>field</code> <code>str</code> <p>Field to set.</p> required <code>value</code> <code>None | str</code> <p>Value to field to. If <code>None</code> will ask user for input.</p> required <code>check_with_message</code> <code>Callable[[str], str]</code> <p>Function which validates <code>value</code> or user input, and returns either an error message if it is invalid, or a False-like value if <code>value</code> is valid.</p> required <code>intro_message</code> <code>str</code> <p>Message that is printed once if user input is requested (e.g. instructions).</p> required <code>input_message</code> <code>str</code> <p>Message that comes with the input prompt.</p> required <code>sanitize</code> <code>Callable[[str], str] | None</code> <p>A function to convert user input to 'more acceptable' input, e.g. for auto-complete. If no correction of user input is possible, return the original value. If no function is provided, don't attempt to correct/auto-complete input.</p> <code>None</code> Source code in <code>openml/cli.py</code> <pre><code>def configure_field(  # noqa: PLR0913\n    field: str,\n    value: None | str,\n    check_with_message: Callable[[str], str],\n    intro_message: str,\n    input_message: str,\n    sanitize: Callable[[str], str] | None = None,\n) -&gt; None:\n    \"\"\"Configure `field` with `value`. If `value` is None ask the user for input.\n\n    `value` and user input are first corrected/auto-completed with `convert_value` if provided,\n    then validated with `check_with_message` function.\n    If the user input a wrong value in interactive mode, the user gets to input a new value.\n    The new valid value is saved in the openml configuration file.\n    In case an invalid `value` is supplied directly (non-interactive), no changes are made.\n\n    Parameters\n    ----------\n    field: str\n        Field to set.\n    value: str, None\n        Value to field to. If `None` will ask user for input.\n    check_with_message: Callable[[str], str]\n        Function which validates `value` or user input, and returns either an error message if it\n        is invalid, or a False-like value if `value` is valid.\n    intro_message: str\n        Message that is printed once if user input is requested (e.g. instructions).\n    input_message: str\n        Message that comes with the input prompt.\n    sanitize: Union[Callable[[str], str], None]\n        A function to convert user input to 'more acceptable' input, e.g. for auto-complete.\n        If no correction of user input is possible, return the original value.\n        If no function is provided, don't attempt to correct/auto-complete input.\n    \"\"\"\n    if value is not None:\n        if sanitize:\n            value = sanitize(value)\n        malformed_input = check_with_message(value)\n        if malformed_input:\n            print(malformed_input)\n            sys.exit()\n    else:\n        print(intro_message)\n        value = wait_until_valid_input(\n            prompt=input_message,\n            check=check_with_message,\n            sanitize=sanitize,\n        )\n    verbose_set(field, value)\n</code></pre>"},{"location":"reference/cli/#openml.cli.wait_until_valid_input","title":"<code>wait_until_valid_input(prompt, check, sanitize)</code>","text":"<p>Asks <code>prompt</code> until an input is received which returns True for <code>check</code>.</p> <p>Parameters:</p> Name Type Description Default <code>prompt</code> <code>str</code> <p>message to display</p> required <code>check</code> <code>Callable[[str], str]</code> <p>function to call with the given input, that provides an error message if the input is not valid otherwise, and False-like otherwise.</p> required <code>sanitize</code> <code>Callable[[str], str] | None</code> <p>A function which attempts to sanitize the user input (e.g. auto-complete).</p> required <p>Returns:</p> Type Description <code>valid input</code> Source code in <code>openml/cli.py</code> <pre><code>def wait_until_valid_input(\n    prompt: str,\n    check: Callable[[str], str],\n    sanitize: Callable[[str], str] | None,\n) -&gt; str:\n    \"\"\"Asks `prompt` until an input is received which returns True for `check`.\n\n    Parameters\n    ----------\n    prompt: str\n        message to display\n    check: Callable[[str], str]\n        function to call with the given input, that provides an error message if the input is not\n        valid otherwise, and False-like otherwise.\n    sanitize: Callable[[str], str], optional\n        A function which attempts to sanitize the user input (e.g. auto-complete).\n\n    Returns\n    -------\n    valid input\n\n    \"\"\"\n    while True:\n        response = input(prompt)\n        if sanitize:\n            response = sanitize(response)\n        error_message = check(response)\n        if error_message:\n            print(error_message, end=\"\\n\\n\")\n        else:\n            return response\n</code></pre>"},{"location":"reference/config/","title":"config","text":"<p>Store module level information like the API key, cache directory and the server</p>"},{"location":"reference/config/#openml.config.ConfigurationForExamples","title":"<code>ConfigurationForExamples</code>","text":"<p>Allows easy switching to and from a test configuration, used for examples.</p> Source code in <code>openml/config.py</code> <pre><code>class ConfigurationForExamples:\n    \"\"\"Allows easy switching to and from a test configuration, used for examples.\"\"\"\n\n    _last_used_server = None\n    _last_used_key = None\n    _start_last_called = False\n    _test_server = \"https://test.openml.org/api/v1/xml\"\n    _test_apikey = \"c0c42819af31e706efe1f4b88c23c6c1\"\n\n    @classmethod\n    def start_using_configuration_for_example(cls) -&gt; None:\n        \"\"\"Sets the configuration to connect to the test server with valid apikey.\n\n        To configuration as was before this call is stored, and can be recovered\n        by using the `stop_use_example_configuration` method.\n        \"\"\"\n        global server  # noqa: PLW0603\n        global apikey  # noqa: PLW0603\n\n        if cls._start_last_called and server == cls._test_server and apikey == cls._test_apikey:\n            # Method is called more than once in a row without modifying the server or apikey.\n            # We don't want to save the current test configuration as a last used configuration.\n            return\n\n        cls._last_used_server = server\n        cls._last_used_key = apikey\n        cls._start_last_called = True\n\n        # Test server key for examples\n        server = cls._test_server\n        apikey = cls._test_apikey\n        warnings.warn(\n            f\"Switching to the test server {server} to not upload results to the live server. \"\n            \"Using the test server may result in reduced performance of the API!\",\n            stacklevel=2,\n        )\n\n    @classmethod\n    def stop_using_configuration_for_example(cls) -&gt; None:\n        \"\"\"Return to configuration as it was before `start_use_example_configuration`.\"\"\"\n        if not cls._start_last_called:\n            # We don't want to allow this because it will (likely) result in the `server` and\n            # `apikey` variables being set to None.\n            raise RuntimeError(\n                \"`stop_use_example_configuration` called without a saved config.\"\n                \"`start_use_example_configuration` must be called first.\",\n            )\n\n        global server  # noqa: PLW0603\n        global apikey  # noqa: PLW0603\n\n        server = cast(str, cls._last_used_server)\n        apikey = cast(str, cls._last_used_key)\n        cls._start_last_called = False\n</code></pre>"},{"location":"reference/config/#openml.config.ConfigurationForExamples.start_using_configuration_for_example","title":"<code>start_using_configuration_for_example()</code>  <code>classmethod</code>","text":"<p>Sets the configuration to connect to the test server with valid apikey.</p> <p>To configuration as was before this call is stored, and can be recovered by using the <code>stop_use_example_configuration</code> method.</p> Source code in <code>openml/config.py</code> <pre><code>@classmethod\ndef start_using_configuration_for_example(cls) -&gt; None:\n    \"\"\"Sets the configuration to connect to the test server with valid apikey.\n\n    To configuration as was before this call is stored, and can be recovered\n    by using the `stop_use_example_configuration` method.\n    \"\"\"\n    global server  # noqa: PLW0603\n    global apikey  # noqa: PLW0603\n\n    if cls._start_last_called and server == cls._test_server and apikey == cls._test_apikey:\n        # Method is called more than once in a row without modifying the server or apikey.\n        # We don't want to save the current test configuration as a last used configuration.\n        return\n\n    cls._last_used_server = server\n    cls._last_used_key = apikey\n    cls._start_last_called = True\n\n    # Test server key for examples\n    server = cls._test_server\n    apikey = cls._test_apikey\n    warnings.warn(\n        f\"Switching to the test server {server} to not upload results to the live server. \"\n        \"Using the test server may result in reduced performance of the API!\",\n        stacklevel=2,\n    )\n</code></pre>"},{"location":"reference/config/#openml.config.ConfigurationForExamples.stop_using_configuration_for_example","title":"<code>stop_using_configuration_for_example()</code>  <code>classmethod</code>","text":"<p>Return to configuration as it was before <code>start_use_example_configuration</code>.</p> Source code in <code>openml/config.py</code> <pre><code>@classmethod\ndef stop_using_configuration_for_example(cls) -&gt; None:\n    \"\"\"Return to configuration as it was before `start_use_example_configuration`.\"\"\"\n    if not cls._start_last_called:\n        # We don't want to allow this because it will (likely) result in the `server` and\n        # `apikey` variables being set to None.\n        raise RuntimeError(\n            \"`stop_use_example_configuration` called without a saved config.\"\n            \"`start_use_example_configuration` must be called first.\",\n        )\n\n    global server  # noqa: PLW0603\n    global apikey  # noqa: PLW0603\n\n    server = cast(str, cls._last_used_server)\n    apikey = cast(str, cls._last_used_key)\n    cls._start_last_called = False\n</code></pre>"},{"location":"reference/config/#openml.config.get_cache_directory","title":"<code>get_cache_directory()</code>","text":"<p>Get the current cache directory.</p> <p>This gets the cache directory for the current server relative to the root cache directory that can be set via <code>set_root_cache_directory()</code>. The cache directory is the <code>root_cache_directory</code> with additional information on which subdirectory to use based on the server name. By default it is <code>root_cache_directory / org / openml / www</code> for the standard OpenML.org server and is defined as <code>root_cache_directory / top-level domain / second-level domain / hostname</code> ```</p>"},{"location":"reference/config/#openml.config.get_cache_directory--returns","title":"Returns","text":"<p>cachedir : string     The current cache directory.</p> Source code in <code>openml/config.py</code> <pre><code>def get_cache_directory() -&gt; str:\n    \"\"\"Get the current cache directory.\n\n    This gets the cache directory for the current server relative\n    to the root cache directory that can be set via\n    ``set_root_cache_directory()``. The cache directory is the\n    ``root_cache_directory`` with additional information on which\n    subdirectory to use based on the server name. By default it is\n    ``root_cache_directory / org / openml / www`` for the standard\n    OpenML.org server and is defined as\n    ``root_cache_directory / top-level domain / second-level domain /\n    hostname``\n    ```\n\n    Returns\n    -------\n    cachedir : string\n        The current cache directory.\n\n    \"\"\"\n    url_suffix = urlparse(server).netloc\n    reversed_url_suffix = os.sep.join(url_suffix.split(\".\")[::-1])  # noqa: PTH118\n    return os.path.join(_root_cache_directory, reversed_url_suffix)  # noqa: PTH118\n</code></pre>"},{"location":"reference/config/#openml.config.get_server_base_url","title":"<code>get_server_base_url()</code>","text":"<p>Return the base URL of the currently configured server.</p> <p>Turns <code>\"https://www.openml.org/api/v1/xml\"</code> in <code>\"https://www.openml.org/\"</code></p> <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/config.py</code> <pre><code>def get_server_base_url() -&gt; str:\n    \"\"\"Return the base URL of the currently configured server.\n\n    Turns ``\"https://www.openml.org/api/v1/xml\"`` in ``\"https://www.openml.org/\"``\n\n    Returns\n    -------\n    str\n    \"\"\"\n    return server.split(\"/api\")[0]\n</code></pre>"},{"location":"reference/config/#openml.config.set_console_log_level","title":"<code>set_console_log_level(console_output_level)</code>","text":"<p>Set console output to the desired level and register it with openml logger if needed.</p> Source code in <code>openml/config.py</code> <pre><code>def set_console_log_level(console_output_level: int) -&gt; None:\n    \"\"\"Set console output to the desired level and register it with openml logger if needed.\"\"\"\n    global console_handler  # noqa: PLW0602\n    assert console_handler is not None\n    _set_level_register_and_store(console_handler, console_output_level)\n</code></pre>"},{"location":"reference/config/#openml.config.set_field_in_config_file","title":"<code>set_field_in_config_file(field, value)</code>","text":"<p>Overwrites the <code>field</code> in the configuration file with the new <code>value</code>.</p> Source code in <code>openml/config.py</code> <pre><code>def set_field_in_config_file(field: str, value: Any) -&gt; None:\n    \"\"\"Overwrites the `field` in the configuration file with the new `value`.\"\"\"\n    if field not in _defaults:\n        raise ValueError(f\"Field '{field}' is not valid and must be one of '{_defaults.keys()}'.\")\n\n    # TODO(eddiebergman): This use of globals has gone too far\n    globals()[field] = value\n    config_file = determine_config_file_path()\n    config = _parse_config(config_file)\n    with config_file.open(\"w\") as fh:\n        for f in _defaults:\n            # We can't blindly set all values based on globals() because when the user\n            # sets it through config.FIELD it should not be stored to file.\n            # There doesn't seem to be a way to avoid writing defaults to file with configparser,\n            # because it is impossible to distinguish from an explicitly set value that matches\n            # the default value, to one that was set to its default because it was omitted.\n            value = config.get(\"FAKE_SECTION\", f)  # type: ignore\n            if f == field:\n                value = globals()[f]\n            fh.write(f\"{f} = {value}\\n\")\n</code></pre>"},{"location":"reference/config/#openml.config.set_file_log_level","title":"<code>set_file_log_level(file_output_level)</code>","text":"<p>Set file output to the desired level and register it with openml logger if needed.</p> Source code in <code>openml/config.py</code> <pre><code>def set_file_log_level(file_output_level: int) -&gt; None:\n    \"\"\"Set file output to the desired level and register it with openml logger if needed.\"\"\"\n    global file_handler  # noqa: PLW0602\n    assert file_handler is not None\n    _set_level_register_and_store(file_handler, file_output_level)\n</code></pre>"},{"location":"reference/config/#openml.config.set_root_cache_directory","title":"<code>set_root_cache_directory(root_cache_directory)</code>","text":"<p>Set module-wide base cache directory.</p> <p>Sets the root cache directory, wherin the cache directories are created to store content from different OpenML servers. For example, by default, cached data for the standard OpenML.org server is stored at <code>root_cache_directory / org / openml / www</code>, and the general pattern is <code>root_cache_directory / top-level domain / second-level domain / hostname</code>.</p> <p>Parameters:</p> Name Type Description Default <code>root_cache_directory</code> <code>string</code> <p>Path to use as cache directory.</p> required See Also <p>get_cache_directory</p> Source code in <code>openml/config.py</code> <pre><code>def set_root_cache_directory(root_cache_directory: str | Path) -&gt; None:\n    \"\"\"Set module-wide base cache directory.\n\n    Sets the root cache directory, wherin the cache directories are\n    created to store content from different OpenML servers. For example,\n    by default, cached data for the standard OpenML.org server is stored\n    at ``root_cache_directory / org / openml / www``, and the general\n    pattern is ``root_cache_directory / top-level domain / second-level\n    domain / hostname``.\n\n    Parameters\n    ----------\n    root_cache_directory : string\n         Path to use as cache directory.\n\n    See Also\n    --------\n    get_cache_directory\n    \"\"\"\n    global _root_cache_directory  # noqa: PLW0603\n    _root_cache_directory = Path(root_cache_directory)\n</code></pre>"},{"location":"reference/exceptions/","title":"exceptions","text":""},{"location":"reference/exceptions/#openml.exceptions.ObjectNotPublishedError","title":"<code>ObjectNotPublishedError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Indicates an object has not been published yet.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class ObjectNotPublishedError(PyOpenMLError):\n    \"\"\"Indicates an object has not been published yet.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLCacheException","title":"<code>OpenMLCacheException</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Dataset / task etc not found in cache</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLCacheException(PyOpenMLError):  # noqa: N818\n    \"\"\"Dataset / task etc not found in cache\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLHashException","title":"<code>OpenMLHashException</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Locally computed hash is different than hash announced by the server.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLHashException(PyOpenMLError):  # noqa: N818\n    \"\"\"Locally computed hash is different than hash announced by the server.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLNotAuthorizedError","title":"<code>OpenMLNotAuthorizedError</code>","text":"<p>               Bases: <code>OpenMLServerError</code></p> <p>Indicates an authenticated user is not authorized to execute the requested action.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLNotAuthorizedError(OpenMLServerError):\n    \"\"\"Indicates an authenticated user is not authorized to execute the requested action.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLPrivateDatasetError","title":"<code>OpenMLPrivateDatasetError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Exception thrown when the user has no rights to access the dataset.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLPrivateDatasetError(PyOpenMLError):\n    \"\"\"Exception thrown when the user has no rights to access the dataset.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLRunsExistError","title":"<code>OpenMLRunsExistError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Indicates run(s) already exists on the server when they should not be duplicated.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLRunsExistError(PyOpenMLError):\n    \"\"\"Indicates run(s) already exists on the server when they should not be duplicated.\"\"\"\n\n    def __init__(self, run_ids: set[int], message: str) -&gt; None:\n        if len(run_ids) &lt; 1:\n            raise ValueError(\"Set of run ids must be non-empty.\")\n        self.run_ids = run_ids\n        super().__init__(message)\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLServerError","title":"<code>OpenMLServerError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>class for when something is really wrong on the server (result did not parse to dict), contains unparsed error.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLServerError(PyOpenMLError):\n    \"\"\"class for when something is really wrong on the server\n    (result did not parse to dict), contains unparsed error.\n    \"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLServerException","title":"<code>OpenMLServerException</code>","text":"<p>               Bases: <code>OpenMLServerError</code></p> <p>exception for when the result of the server was not 200 (e.g., listing call w/o results).</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLServerException(OpenMLServerError):  # noqa: N818\n    \"\"\"exception for when the result of the server was\n    not 200 (e.g., listing call w/o results).\n    \"\"\"\n\n    # Code needs to be optional to allow the exception to be picklable:\n    # https://stackoverflow.com/questions/16244923/how-to-make-a-custom-exception-class-with-multiple-init-args-pickleable  # noqa: E501\n    def __init__(self, message: str, code: int | None = None, url: str | None = None):\n        self.message = message\n        self.code = code\n        self.url = url\n        super().__init__(message)\n\n    def __str__(self) -&gt; str:\n        return f\"{self.url} returned code {self.code}: {self.message}\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLServerNoResult","title":"<code>OpenMLServerNoResult</code>","text":"<p>               Bases: <code>OpenMLServerException</code></p> <p>Exception for when the result of the server is empty.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLServerNoResult(OpenMLServerException):\n    \"\"\"Exception for when the result of the server is empty.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.PyOpenMLError","title":"<code>PyOpenMLError</code>","text":"<p>               Bases: <code>Exception</code></p> <p>Base class for all exceptions in OpenML-Python.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class PyOpenMLError(Exception):\n    \"\"\"Base class for all exceptions in OpenML-Python.\"\"\"\n\n    def __init__(self, message: str):\n        self.message = message\n        super().__init__(message)\n</code></pre>"},{"location":"reference/testing/","title":"testing","text":""},{"location":"reference/testing/#openml.testing.CustomImputer","title":"<code>CustomImputer</code>","text":"<p>               Bases: <code>Imputer</code></p> <p>Duplicate class alias for sklearn's SimpleImputer</p> <p>Helps bypass the sklearn extension duplicate operation check</p> Source code in <code>openml/testing.py</code> <pre><code>class CustomImputer(SimpleImputer):\n    \"\"\"Duplicate class alias for sklearn's SimpleImputer\n\n    Helps bypass the sklearn extension duplicate operation check\n    \"\"\"\n</code></pre>"},{"location":"reference/testing/#openml.testing.TestBase","title":"<code>TestBase</code>","text":"<p>               Bases: <code>TestCase</code></p> <p>Base class for tests</p> Note <p>Currently hard-codes a read-write key. Hopefully soon allows using a test server, not the production server.</p> Source code in <code>openml/testing.py</code> <pre><code>class TestBase(unittest.TestCase):\n    \"\"\"Base class for tests\n\n    Note\n    ----\n    Currently hard-codes a read-write key.\n    Hopefully soon allows using a test server, not the production server.\n    \"\"\"\n\n    # TODO: This could be made more explcit with a TypedDict instead of list[str | int]\n    publish_tracker: ClassVar[dict[str, list[str | int]]] = {\n        \"run\": [],\n        \"data\": [],\n        \"flow\": [],\n        \"task\": [],\n        \"study\": [],\n        \"user\": [],\n    }\n    flow_name_tracker: ClassVar[list[str]] = []\n    test_server = \"https://test.openml.org/api/v1/xml\"\n    # amueller's read/write key that he will throw away later\n    apikey = \"610344db6388d9ba34f6db45a3cf71de\"\n\n    # creating logger for tracking files uploaded to test server\n    logger = logging.getLogger(\"unit_tests_published_entities\")\n    logger.setLevel(logging.DEBUG)\n\n    def setUp(self, n_levels: int = 1) -&gt; None:\n        \"\"\"Setup variables and temporary directories.\n\n        In particular, this methods:\n\n        * creates a temporary working directory\n        * figures out a path to a few static test files\n        * set the default server to be the test server\n        * set a static API key for the test server\n        * increases the maximal number of retries\n\n        Parameters\n        ----------\n        n_levels : int\n            Number of nested directories the test is in. Necessary to resolve the path to the\n            ``files`` directory, which is located directly under the ``tests`` directory.\n        \"\"\"\n        # This cache directory is checked in to git to simulate a populated\n        # cache\n        self.maxDiff = None\n        abspath_this_file = Path(inspect.getfile(self.__class__)).absolute()\n        static_cache_dir = abspath_this_file.parent\n        for _ in range(n_levels):\n            static_cache_dir = static_cache_dir.parent.absolute()\n\n        content = os.listdir(static_cache_dir)\n        if \"files\" in content:\n            static_cache_dir = static_cache_dir / \"files\"\n        else:\n            raise ValueError(\n                f\"Cannot find test cache dir, expected it to be {static_cache_dir}!\",\n            )\n\n        self.static_cache_dir = static_cache_dir\n        self.cwd = Path.cwd()\n        workdir = Path(__file__).parent.absolute()\n        tmp_dir_name = self.id()\n        self.workdir = workdir / tmp_dir_name\n        shutil.rmtree(self.workdir, ignore_errors=True)\n\n        self.workdir.mkdir(exist_ok=True)\n        os.chdir(self.workdir)\n\n        self.cached = True\n        openml.config.apikey = TestBase.apikey\n        self.production_server = \"https://openml.org/api/v1/xml\"\n        openml.config.server = TestBase.test_server\n        openml.config.avoid_duplicate_runs = False\n        openml.config.set_root_cache_directory(str(self.workdir))\n\n        # Increase the number of retries to avoid spurious server failures\n        self.retry_policy = openml.config.retry_policy\n        self.connection_n_retries = openml.config.connection_n_retries\n        openml.config.set_retry_policy(\"robot\", n_retries=20)\n\n    def tearDown(self) -&gt; None:\n        \"\"\"Tear down the test\"\"\"\n        os.chdir(self.cwd)\n        try:\n            shutil.rmtree(self.workdir)\n        except PermissionError as e:\n            if os.name != \"nt\":\n                # one of the files may still be used by another process\n                raise e\n\n        openml.config.server = self.production_server\n        openml.config.connection_n_retries = self.connection_n_retries\n        openml.config.retry_policy = self.retry_policy\n\n    @classmethod\n    def _mark_entity_for_removal(\n        cls,\n        entity_type: str,\n        entity_id: int,\n        entity_name: str | None = None,\n    ) -&gt; None:\n        \"\"\"Static record of entities uploaded to test server\n\n        Dictionary of lists where the keys are 'entity_type'.\n        Each such dictionary is a list of integer IDs.\n        For entity_type='flow', each list element is a tuple\n        of the form (Flow ID, Flow Name).\n        \"\"\"\n        if entity_type not in TestBase.publish_tracker:\n            TestBase.publish_tracker[entity_type] = [entity_id]\n        else:\n            TestBase.publish_tracker[entity_type].append(entity_id)\n        if isinstance(entity_type, openml.flows.OpenMLFlow):\n            assert entity_name is not None\n            cls.flow_name_tracker.append(entity_name)\n\n    @classmethod\n    def _delete_entity_from_tracker(cls, entity_type: str, entity: int) -&gt; None:\n        \"\"\"Deletes entity records from the static file_tracker\n\n        Given an entity type and corresponding ID, deletes all entries, including\n        duplicate entries of the ID for the entity type.\n        \"\"\"\n        if entity_type in TestBase.publish_tracker:\n            # removes duplicate entries\n            TestBase.publish_tracker[entity_type] = list(set(TestBase.publish_tracker[entity_type]))\n            if entity_type == \"flow\":\n                delete_index = next(\n                    i\n                    for i, (id_, _) in enumerate(\n                        zip(TestBase.publish_tracker[entity_type], TestBase.flow_name_tracker),\n                    )\n                    if id_ == entity\n                )\n            else:\n                delete_index = next(\n                    i\n                    for i, id_ in enumerate(TestBase.publish_tracker[entity_type])\n                    if id_ == entity\n                )\n            TestBase.publish_tracker[entity_type].pop(delete_index)\n\n    def _get_sentinel(self, sentinel: str | None = None) -&gt; str:\n        if sentinel is None:\n            # Create a unique prefix for the flow. Necessary because the flow\n            # is identified by its name and external version online. Having a\n            # unique name allows us to publish the same flow in each test run.\n            md5 = hashlib.md5()  # noqa: S324\n            md5.update(str(time.time()).encode(\"utf-8\"))\n            md5.update(str(os.getpid()).encode(\"utf-8\"))\n            sentinel = md5.hexdigest()[:10]\n            sentinel = \"TEST%s\" % sentinel\n        return sentinel\n\n    def _add_sentinel_to_flow_name(\n        self,\n        flow: openml.flows.OpenMLFlow,\n        sentinel: str | None = None,\n    ) -&gt; tuple[openml.flows.OpenMLFlow, str]:\n        sentinel = self._get_sentinel(sentinel=sentinel)\n        flows_to_visit = []\n        flows_to_visit.append(flow)\n        while len(flows_to_visit) &gt; 0:\n            current_flow = flows_to_visit.pop()\n            current_flow.name = f\"{sentinel}{current_flow.name}\"\n            for subflow in current_flow.components.values():\n                flows_to_visit.append(subflow)\n\n        return flow, sentinel\n\n    def _check_dataset(self, dataset: dict[str, str | int]) -&gt; None:\n        _check_dataset(dataset)\n        assert isinstance(dataset, dict)\n        assert len(dataset) &gt;= 2\n        assert \"did\" in dataset\n        assert isinstance(dataset[\"did\"], int)\n        assert \"status\" in dataset\n        assert isinstance(dataset[\"status\"], str)\n        assert dataset[\"status\"] in [\"in_preparation\", \"active\", \"deactivated\"]\n\n    def _check_fold_timing_evaluations(  # noqa: PLR0913\n        self,\n        fold_evaluations: dict[str, dict[int, dict[int, float]]],\n        num_repeats: int,\n        num_folds: int,\n        *,\n        max_time_allowed: float = 60000.0,\n        task_type: TaskType = TaskType.SUPERVISED_CLASSIFICATION,\n        check_scores: bool = True,\n    ) -&gt; None:\n        \"\"\"\n        Checks whether the right timing measures are attached to the run\n        (before upload). Test is only performed for versions &gt;= Python3.3\n\n        In case of check_n_jobs(clf) == false, please do not perform this\n        check (check this condition outside of this function. )\n        default max_time_allowed (per fold, in milli seconds) = 1 minute,\n        quite pessimistic\n        \"\"\"\n        # a dict mapping from openml measure to a tuple with the minimum and\n        # maximum allowed value\n        check_measures = {\n            # should take at least one millisecond (?)\n            \"usercpu_time_millis_testing\": (0, max_time_allowed),\n            \"usercpu_time_millis_training\": (0, max_time_allowed),\n            \"usercpu_time_millis\": (0, max_time_allowed),\n            \"wall_clock_time_millis_training\": (0, max_time_allowed),\n            \"wall_clock_time_millis_testing\": (0, max_time_allowed),\n            \"wall_clock_time_millis\": (0, max_time_allowed),\n        }\n\n        if check_scores:\n            if task_type in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n                check_measures[\"predictive_accuracy\"] = (0, 1.0)\n            elif task_type == TaskType.SUPERVISED_REGRESSION:\n                check_measures[\"mean_absolute_error\"] = (0, float(\"inf\"))\n\n        assert isinstance(fold_evaluations, dict)\n        assert set(fold_evaluations.keys()) == set(check_measures.keys())\n\n        for measure in check_measures:\n            if measure in fold_evaluations:\n                num_rep_entrees = len(fold_evaluations[measure])\n                assert num_rep_entrees == num_repeats\n                min_val = check_measures[measure][0]\n                max_val = check_measures[measure][1]\n                for rep in range(num_rep_entrees):\n                    num_fold_entrees = len(fold_evaluations[measure][rep])\n                    assert num_fold_entrees == num_folds\n                    for fold in range(num_fold_entrees):\n                        evaluation = fold_evaluations[measure][rep][fold]\n                        assert isinstance(evaluation, float)\n                        assert evaluation &gt;= min_val\n                        assert evaluation &lt;= max_val\n</code></pre>"},{"location":"reference/testing/#openml.testing.TestBase.setUp","title":"<code>setUp(n_levels=1)</code>","text":"<p>Setup variables and temporary directories.</p> <p>In particular, this methods:</p> <ul> <li>creates a temporary working directory</li> <li>figures out a path to a few static test files</li> <li>set the default server to be the test server</li> <li>set a static API key for the test server</li> <li>increases the maximal number of retries</li> </ul> <p>Parameters:</p> Name Type Description Default <code>n_levels</code> <code>int</code> <p>Number of nested directories the test is in. Necessary to resolve the path to the <code>files</code> directory, which is located directly under the <code>tests</code> directory.</p> <code>1</code> Source code in <code>openml/testing.py</code> <pre><code>def setUp(self, n_levels: int = 1) -&gt; None:\n    \"\"\"Setup variables and temporary directories.\n\n    In particular, this methods:\n\n    * creates a temporary working directory\n    * figures out a path to a few static test files\n    * set the default server to be the test server\n    * set a static API key for the test server\n    * increases the maximal number of retries\n\n    Parameters\n    ----------\n    n_levels : int\n        Number of nested directories the test is in. Necessary to resolve the path to the\n        ``files`` directory, which is located directly under the ``tests`` directory.\n    \"\"\"\n    # This cache directory is checked in to git to simulate a populated\n    # cache\n    self.maxDiff = None\n    abspath_this_file = Path(inspect.getfile(self.__class__)).absolute()\n    static_cache_dir = abspath_this_file.parent\n    for _ in range(n_levels):\n        static_cache_dir = static_cache_dir.parent.absolute()\n\n    content = os.listdir(static_cache_dir)\n    if \"files\" in content:\n        static_cache_dir = static_cache_dir / \"files\"\n    else:\n        raise ValueError(\n            f\"Cannot find test cache dir, expected it to be {static_cache_dir}!\",\n        )\n\n    self.static_cache_dir = static_cache_dir\n    self.cwd = Path.cwd()\n    workdir = Path(__file__).parent.absolute()\n    tmp_dir_name = self.id()\n    self.workdir = workdir / tmp_dir_name\n    shutil.rmtree(self.workdir, ignore_errors=True)\n\n    self.workdir.mkdir(exist_ok=True)\n    os.chdir(self.workdir)\n\n    self.cached = True\n    openml.config.apikey = TestBase.apikey\n    self.production_server = \"https://openml.org/api/v1/xml\"\n    openml.config.server = TestBase.test_server\n    openml.config.avoid_duplicate_runs = False\n    openml.config.set_root_cache_directory(str(self.workdir))\n\n    # Increase the number of retries to avoid spurious server failures\n    self.retry_policy = openml.config.retry_policy\n    self.connection_n_retries = openml.config.connection_n_retries\n    openml.config.set_retry_policy(\"robot\", n_retries=20)\n</code></pre>"},{"location":"reference/testing/#openml.testing.TestBase.tearDown","title":"<code>tearDown()</code>","text":"<p>Tear down the test</p> Source code in <code>openml/testing.py</code> <pre><code>def tearDown(self) -&gt; None:\n    \"\"\"Tear down the test\"\"\"\n    os.chdir(self.cwd)\n    try:\n        shutil.rmtree(self.workdir)\n    except PermissionError as e:\n        if os.name != \"nt\":\n            # one of the files may still be used by another process\n            raise e\n\n    openml.config.server = self.production_server\n    openml.config.connection_n_retries = self.connection_n_retries\n    openml.config.retry_policy = self.retry_policy\n</code></pre>"},{"location":"reference/testing/#openml.testing.check_task_existence","title":"<code>check_task_existence(task_type, dataset_id, target_name, **kwargs)</code>","text":"<p>Checks if any task with exists on test server that matches the meta data.</p> Parameter <p>task_type : openml.tasks.TaskType dataset_id : int target_name : str</p> Return <p>int, None</p> Source code in <code>openml/testing.py</code> <pre><code>def check_task_existence(\n    task_type: TaskType,\n    dataset_id: int,\n    target_name: str,\n    **kwargs: dict[str, str | int | dict[str, str | int | openml.tasks.TaskType]],\n) -&gt; int | None:\n    \"\"\"Checks if any task with exists on test server that matches the meta data.\n\n    Parameter\n    ---------\n    task_type : openml.tasks.TaskType\n    dataset_id : int\n    target_name : str\n\n    Return\n    ------\n    int, None\n    \"\"\"\n    return_val = None\n    tasks = openml.tasks.list_tasks(task_type=task_type, output_format=\"dataframe\")\n    assert isinstance(tasks, pd.DataFrame)\n    if len(tasks) == 0:\n        return None\n    tasks = tasks.loc[tasks[\"did\"] == dataset_id]\n    if len(tasks) == 0:\n        return None\n    tasks = tasks.loc[tasks[\"target_feature\"] == target_name]\n    if len(tasks) == 0:\n        return None\n    task_match = []\n    for task_id in tasks[\"tid\"].to_list():\n        task_match.append(task_id)\n        try:\n            task = openml.tasks.get_task(task_id)\n        except OpenMLServerException:\n            # can fail if task_id deleted by another parallely run unit test\n            task_match.pop(-1)\n            return_val = None\n            continue\n        for k, v in kwargs.items():\n            if getattr(task, k) != v:\n                # even if one of the meta-data key mismatches, then task_id is not a match\n                task_match.pop(-1)\n                break\n        # if task_id is retained in the task_match list, it passed all meta key-value matches\n        if len(task_match) == 1:\n            return_val = task_id\n            break\n    if len(task_match) == 0:\n        return_val = None\n    return return_val\n</code></pre>"},{"location":"reference/utils/","title":"utils","text":""},{"location":"reference/utils/#openml.utils.extract_xml_tags","title":"<code>extract_xml_tags(xml_tag_name, node, *, allow_none=True)</code>","text":"<pre><code>extract_xml_tags(xml_tag_name: str, node: Mapping[str, Any], *, allow_none: Literal[True] = ...) -&gt; Any | None\n</code></pre><pre><code>extract_xml_tags(xml_tag_name: str, node: Mapping[str, Any], *, allow_none: Literal[False]) -&gt; Any\n</code></pre> <p>Helper to extract xml tags from xmltodict.</p> <p>Parameters:</p> Name Type Description Default <code>xml_tag_name</code> <code>str</code> <p>Name of the xml tag to extract from the node.</p> required <code>node</code> <code>Mapping[str, Any]</code> <p>Node object returned by <code>xmltodict</code> from which <code>xml_tag_name</code> should be extracted.</p> required <code>allow_none</code> <code>bool</code> <p>If <code>False</code>, the tag needs to exist in the node. Will raise a <code>ValueError</code> if it does not.</p> <code>True</code> <p>Returns:</p> Type Description <code>object</code> Source code in <code>openml/utils.py</code> <pre><code>def extract_xml_tags(\n    xml_tag_name: str,\n    node: Mapping[str, Any],\n    *,\n    allow_none: bool = True,\n) -&gt; Any | None:\n    \"\"\"Helper to extract xml tags from xmltodict.\n\n    Parameters\n    ----------\n    xml_tag_name : str\n        Name of the xml tag to extract from the node.\n\n    node : Mapping[str, Any]\n        Node object returned by ``xmltodict`` from which ``xml_tag_name``\n        should be extracted.\n\n    allow_none : bool\n        If ``False``, the tag needs to exist in the node. Will raise a\n        ``ValueError`` if it does not.\n\n    Returns\n    -------\n    object\n    \"\"\"\n    if xml_tag_name in node and node[xml_tag_name] is not None:\n        if isinstance(node[xml_tag_name], (dict, str)):\n            return [node[xml_tag_name]]\n        if isinstance(node[xml_tag_name], list):\n            return node[xml_tag_name]\n\n        raise ValueError(\"Received not string and non list as tag item\")\n\n    if allow_none:\n        return None\n\n    raise ValueError(f\"Could not find tag '{xml_tag_name}' in node '{node!s}'\")\n</code></pre>"},{"location":"reference/datasets/","title":"datasets","text":""},{"location":"reference/datasets/#openml.datasets.OpenMLDataFeature","title":"<code>OpenMLDataFeature</code>","text":"<p>Data Feature (a.k.a. Attribute) object.</p> <p>Parameters:</p> Name Type Description Default <code>index</code> <code>int</code> <p>The index of this feature</p> required <code>name</code> <code>str</code> <p>Name of the feature</p> required <code>data_type</code> <code>str</code> <p>can be nominal, numeric, string, date (corresponds to arff)</p> required <code>nominal_values</code> <code>list(str)</code> <p>list of the possible values, in case of nominal attribute</p> required <code>number_missing_values</code> <code>int</code> <p>Number of rows that have a missing value for this feature.</p> required <code>ontologies</code> <code>list(str)</code> <p>list of ontologies attached to this feature. An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided.</p> <code>None</code> Source code in <code>openml/datasets/data_feature.py</code> <pre><code>class OpenMLDataFeature:\n    \"\"\"\n    Data Feature (a.k.a. Attribute) object.\n\n    Parameters\n    ----------\n    index : int\n        The index of this feature\n    name : str\n        Name of the feature\n    data_type : str\n        can be nominal, numeric, string, date (corresponds to arff)\n    nominal_values : list(str)\n        list of the possible values, in case of nominal attribute\n    number_missing_values : int\n        Number of rows that have a missing value for this feature.\n    ontologies : list(str)\n        list of ontologies attached to this feature. An ontology describes the\n        concept that are described in a feature. An ontology is defined by an\n        URL where the information is provided.\n    \"\"\"\n\n    LEGAL_DATA_TYPES: ClassVar[Sequence[str]] = [\"nominal\", \"numeric\", \"string\", \"date\"]\n\n    def __init__(  # noqa: PLR0913\n        self,\n        index: int,\n        name: str,\n        data_type: str,\n        nominal_values: list[str],\n        number_missing_values: int,\n        ontologies: list[str] | None = None,\n    ):\n        if not isinstance(index, int):\n            raise TypeError(f\"Index must be `int` but is {type(index)}\")\n\n        if data_type not in self.LEGAL_DATA_TYPES:\n            raise ValueError(\n                f\"data type should be in {self.LEGAL_DATA_TYPES!s}, found: {data_type}\",\n            )\n\n        if data_type == \"nominal\":\n            if nominal_values is None:\n                raise TypeError(\n                    \"Dataset features require attribute `nominal_values` for nominal \"\n                    \"feature type.\",\n                )\n\n            if not isinstance(nominal_values, list):\n                raise TypeError(\n                    \"Argument `nominal_values` is of wrong datatype, should be list, \"\n                    f\"but is {type(nominal_values)}\",\n                )\n        elif nominal_values is not None:\n            raise TypeError(\"Argument `nominal_values` must be None for non-nominal feature.\")\n\n        if not isinstance(number_missing_values, int):\n            msg = f\"number_missing_values must be int but is {type(number_missing_values)}\"\n            raise TypeError(msg)\n\n        self.index = index\n        self.name = str(name)\n        self.data_type = str(data_type)\n        self.nominal_values = nominal_values\n        self.number_missing_values = number_missing_values\n        self.ontologies = ontologies\n\n    def __repr__(self) -&gt; str:\n        return \"[%d - %s (%s)]\" % (self.index, self.name, self.data_type)\n\n    def __eq__(self, other: Any) -&gt; bool:\n        return isinstance(other, OpenMLDataFeature) and self.__dict__ == other.__dict__\n\n    def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -&gt; None:  # noqa: FBT001, ARG002\n        pp.text(str(self))\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset","title":"<code>OpenMLDataset</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>Dataset object.</p> <p>Allows fetching and uploading datasets to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>data_format</code> <code>str</code> <p>Format of the dataset which can be either 'arff' or 'sparse_arff'.</p> <code>'arff'</code> <code>cache_format</code> <code>str</code> <p>Format for caching the dataset which can be either 'feather' or 'pickle'.</p> <code>'pickle'</code> <code>dataset_id</code> <code>int</code> <p>Id autogenerated by the server.</p> <code>None</code> <code>version</code> <code>int</code> <p>Version of this dataset. '1' for original version. Auto-incremented by server.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>The date-time when the dataset was uploaded, generated by server.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>licence</code> <code>str</code> <p>License of the data.</p> <code>None</code> <code>url</code> <code>str</code> <p>Valid URL, points to actual data file. The file can be on the OpenML server or another dataset repository.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>version_label</code> <code>str</code> <p>Version label provided by user. Can be a date, hash, or some other type of id.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>tag</code> <code>str</code> <p>Tags, describing the algorithms.</p> <code>None</code> <code>visibility</code> <code>str</code> <p>Who can see the dataset. Typical values: 'Everyone','All my friends','Only me'. Can also be any of the user's circles.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <code>md5_checksum</code> <code>str</code> <p>MD5 checksum to check if the dataset is downloaded without corruption.</p> <code>None</code> <code>data_file</code> <code>str</code> <p>Path to where the dataset is located.</p> <code>None</code> <code>features_file</code> <code>dict</code> <p>A dictionary of dataset features, which maps a feature index to a OpenMLDataFeature.</p> <code>None</code> <code>qualities_file</code> <code>dict</code> <p>A dictionary of dataset qualities, which maps a quality name to a quality value.</p> <code>None</code> <code>dataset</code> <code>str | None</code> <p>Serialized arff dataset string.</p> <code>None</code> <code>parquet_url</code> <code>str | None</code> <p>This is the URL to the storage location where the dataset files are hosted. This can be a MinIO bucket URL. If specified, the data will be accessed from this URL when reading the files.</p> <code>None</code> <code>parquet_file</code> <code>str | None</code> <p>Path to the local file.</p> <code>None</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>class OpenMLDataset(OpenMLBase):\n    \"\"\"Dataset object.\n\n    Allows fetching and uploading datasets to OpenML.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    data_format : str\n        Format of the dataset which can be either 'arff' or 'sparse_arff'.\n    cache_format : str\n        Format for caching the dataset which can be either 'feather' or 'pickle'.\n    dataset_id : int, optional\n        Id autogenerated by the server.\n    version : int, optional\n        Version of this dataset. '1' for original version.\n        Auto-incremented by server.\n    creator : str, optional\n        The person who created the dataset.\n    contributor : str, optional\n        People who contributed to the current version of the dataset.\n    collection_date : str, optional\n        The date the data was originally collected, given by the uploader.\n    upload_date : str, optional\n        The date-time when the dataset was uploaded, generated by server.\n    language : str, optional\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str, optional\n        License of the data.\n    url : str, optional\n        Valid URL, points to actual data file.\n        The file can be on the OpenML server or another dataset repository.\n    default_target_attribute : str, optional\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column,\n        if present in the dataset.\n    ignore_attribute : str | list, optional\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    version_label : str, optional\n        Version label provided by user.\n        Can be a date, hash, or some other type of id.\n    citation : str, optional\n        Reference(s) that should be cited when building on this data.\n    tag : str, optional\n        Tags, describing the algorithms.\n    visibility : str, optional\n        Who can see the dataset.\n        Typical values: 'Everyone','All my friends','Only me'.\n        Can also be any of the user's circles.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n    md5_checksum : str, optional\n        MD5 checksum to check if the dataset is downloaded without corruption.\n    data_file : str, optional\n        Path to where the dataset is located.\n    features_file : dict, optional\n        A dictionary of dataset features,\n        which maps a feature index to a OpenMLDataFeature.\n    qualities_file : dict, optional\n        A dictionary of dataset qualities,\n        which maps a quality name to a quality value.\n    dataset: string, optional\n        Serialized arff dataset string.\n    parquet_url: string, optional\n        This is the URL to the storage location where the dataset files are hosted.\n        This can be a MinIO bucket URL. If specified, the data will be accessed\n        from this URL when reading the files.\n    parquet_file: string, optional\n        Path to the local file.\n    \"\"\"\n\n    def __init__(  # noqa: C901, PLR0912, PLR0913, PLR0915\n        self,\n        name: str,\n        description: str | None,\n        data_format: Literal[\"arff\", \"sparse_arff\"] = \"arff\",\n        cache_format: Literal[\"feather\", \"pickle\"] = \"pickle\",\n        dataset_id: int | None = None,\n        version: int | None = None,\n        creator: str | None = None,\n        contributor: str | None = None,\n        collection_date: str | None = None,\n        upload_date: str | None = None,\n        language: str | None = None,\n        licence: str | None = None,\n        url: str | None = None,\n        default_target_attribute: str | None = None,\n        row_id_attribute: str | None = None,\n        ignore_attribute: str | list[str] | None = None,\n        version_label: str | None = None,\n        citation: str | None = None,\n        tag: str | None = None,\n        visibility: str | None = None,\n        original_data_url: str | None = None,\n        paper_url: str | None = None,\n        update_comment: str | None = None,\n        md5_checksum: str | None = None,\n        data_file: str | None = None,\n        features_file: str | None = None,\n        qualities_file: str | None = None,\n        dataset: str | None = None,\n        parquet_url: str | None = None,\n        parquet_file: str | None = None,\n    ):\n        if cache_format not in [\"feather\", \"pickle\"]:\n            raise ValueError(\n                \"cache_format must be one of 'feather' or 'pickle. \"\n                f\"Invalid format specified: {cache_format}\",\n            )\n\n        def find_invalid_characters(string: str, pattern: str) -&gt; str:\n            invalid_chars = set()\n            regex = re.compile(pattern)\n            for char in string:\n                if not regex.match(char):\n                    invalid_chars.add(char)\n            return \",\".join(\n                [f\"'{char}'\" if char != \"'\" else f'\"{char}\"' for char in invalid_chars],\n            )\n\n        if dataset_id is None:\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if description and not re.match(pattern, description):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(description, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in description: {description}\",\n                )\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if citation and not re.match(pattern, citation):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(citation, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in citation: {citation}\",\n                )\n            pattern = \"^[a-zA-Z0-9_\\\\-\\\\.\\\\(\\\\),]+$\"\n            if not re.match(pattern, name):\n                # regex given by server in error message\n                invalid_characters = find_invalid_characters(name, pattern)\n                raise ValueError(f\"Invalid symbols {invalid_characters} in name: {name}\")\n\n        self.ignore_attribute: list[str] | None = None\n        if isinstance(ignore_attribute, str):\n            self.ignore_attribute = [ignore_attribute]\n        elif isinstance(ignore_attribute, list) or ignore_attribute is None:\n            self.ignore_attribute = ignore_attribute\n        else:\n            raise ValueError(\"Wrong data type for ignore_attribute. Should be list.\")\n\n        # TODO add function to check if the name is casual_string128\n        # Attributes received by querying the RESTful API\n        self.dataset_id = int(dataset_id) if dataset_id is not None else None\n        self.name = name\n        self.version = int(version) if version is not None else None\n        self.description = description\n        self.cache_format = cache_format\n        # Has to be called format, otherwise there will be an XML upload error\n        self.format = data_format\n        self.creator = creator\n        self.contributor = contributor\n        self.collection_date = collection_date\n        self.upload_date = upload_date\n        self.language = language\n        self.licence = licence\n        self.url = url\n        self.default_target_attribute = default_target_attribute\n        self.row_id_attribute = row_id_attribute\n\n        self.version_label = version_label\n        self.citation = citation\n        self.tag = tag\n        self.visibility = visibility\n        self.original_data_url = original_data_url\n        self.paper_url = paper_url\n        self.update_comment = update_comment\n        self.md5_checksum = md5_checksum\n        self.data_file = data_file\n        self.parquet_file = parquet_file\n        self._dataset = dataset\n        self._parquet_url = parquet_url\n\n        self._features: dict[int, OpenMLDataFeature] | None = None\n        self._qualities: dict[str, float] | None = None\n        self._no_qualities_found = False\n\n        if features_file is not None:\n            self._features = _read_features(Path(features_file))\n\n        # \"\" was the old default value by `get_dataset` and maybe still used by some\n        if qualities_file == \"\":\n            # TODO(0.15): to switch to \"qualities_file is not None\" below and remove warning\n            warnings.warn(\n                \"Starting from Version 0.15 `qualities_file` must be None and not an empty string \"\n                \"to avoid reading the qualities from file. Set `qualities_file` to None to avoid \"\n                \"this warning.\",\n                FutureWarning,\n                stacklevel=2,\n            )\n            qualities_file = None\n\n        if qualities_file is not None:\n            self._qualities = _read_qualities(Path(qualities_file))\n\n        if data_file is not None:\n            data_pickle, data_feather, feather_attribute = self._compressed_cache_file_paths(\n                Path(data_file)\n            )\n            self.data_pickle_file = data_pickle if Path(data_pickle).exists() else None\n            self.data_feather_file = data_feather if Path(data_feather).exists() else None\n            self.feather_attribute_file = feather_attribute if Path(feather_attribute) else None\n        else:\n            self.data_pickle_file = None\n            self.data_feather_file = None\n            self.feather_attribute_file = None\n\n    @property\n    def features(self) -&gt; dict[int, OpenMLDataFeature]:\n        \"\"\"Get the features of this dataset.\"\"\"\n        if self._features is None:\n            # TODO(eddiebergman): These should return a value so we can set it to be not None\n            self._load_features()\n\n        assert self._features is not None\n        return self._features\n\n    @property\n    def qualities(self) -&gt; dict[str, float] | None:\n        \"\"\"Get the qualities of this dataset.\"\"\"\n        # TODO(eddiebergman): Better docstring, I don't know what qualities means\n\n        # We have to check `_no_qualities_found` as there might not be qualities for a dataset\n        if self._qualities is None and (not self._no_qualities_found):\n            self._load_qualities()\n\n        return self._qualities\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Get the dataset numeric id.\"\"\"\n        return self.dataset_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Obtain number of features in accordance with lazy loading.\n        n_features: int | None = None\n        if self._qualities is not None and self._qualities[\"NumberOfFeatures\"] is not None:\n            n_features = int(self._qualities[\"NumberOfFeatures\"])\n        elif self._features is not None:\n            n_features = len(self._features)\n\n        fields: dict[str, int | str | None] = {\n            \"Name\": self.name,\n            \"Version\": self.version,\n            \"Format\": self.format,\n            \"Licence\": self.licence,\n            \"Download URL\": self.url,\n            \"Data file\": str(self.data_file) if self.data_file is not None else None,\n            \"Pickle file\": (\n                str(self.data_pickle_file) if self.data_pickle_file is not None else None\n            ),\n            \"# of features\": n_features,\n        }\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.dataset_id is not None:\n            fields[\"OpenML URL\"] = self.openml_url\n        if self._qualities is not None and self._qualities[\"NumberOfInstances\"] is not None:\n            fields[\"# of instances\"] = int(self._qualities[\"NumberOfInstances\"])\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Name\",\n            \"Version\",\n            \"Format\",\n            \"Upload Date\",\n            \"Licence\",\n            \"Download URL\",\n            \"OpenML URL\",\n            \"Data File\",\n            \"Pickle File\",\n            \"# of features\",\n            \"# of instances\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if not isinstance(other, OpenMLDataset):\n            return False\n\n        server_fields = {\n            \"dataset_id\",\n            \"version\",\n            \"upload_date\",\n            \"url\",\n            \"dataset\",\n            \"data_file\",\n        }\n\n        # check that common keys and values are identical\n        self_keys = set(self.__dict__.keys()) - server_fields\n        other_keys = set(other.__dict__.keys()) - server_fields\n        return self_keys == other_keys and all(\n            self.__dict__[key] == other.__dict__[key] for key in self_keys\n        )\n\n    def _download_data(self) -&gt; None:\n        \"\"\"Download ARFF data file to standard cache directory. Set `self.data_file`.\"\"\"\n        # import required here to avoid circular import.\n        from .functions import _get_dataset_arff, _get_dataset_parquet\n\n        self.data_file = str(_get_dataset_arff(self))\n        if self._parquet_url is not None:\n            self.parquet_file = str(_get_dataset_parquet(self))\n\n    def _get_arff(self, format: str) -&gt; dict:  # noqa: A002\n        \"\"\"Read ARFF file and return decoded arff.\n\n        Reads the file referenced in self.data_file.\n\n        Parameters\n        ----------\n        format : str\n            Format of the ARFF file.\n            Must be one of 'arff' or 'sparse_arff' or a string that will be either of those\n            when converted to lower case.\n\n\n\n        Returns\n        -------\n        dict\n            Decoded arff.\n\n        \"\"\"\n        # TODO: add a partial read method which only returns the attribute\n        # headers of the corresponding .arff file!\n        import struct\n\n        filename = self.data_file\n        assert filename is not None\n        filepath = Path(filename)\n\n        bits = 8 * struct.calcsize(\"P\")\n\n        # Files can be considered too large on a 32-bit system,\n        # if it exceeds 120mb (slightly more than covtype dataset size)\n        # This number is somewhat arbitrary.\n        if bits != 64:\n            MB_120 = 120_000_000\n            file_size = filepath.stat().st_size\n            if file_size &gt; MB_120:\n                raise NotImplementedError(\n                    f\"File {filename} too big for {file_size}-bit system ({bits} bytes).\",\n                )\n\n        if format.lower() == \"arff\":\n            return_type = arff.DENSE\n        elif format.lower() == \"sparse_arff\":\n            return_type = arff.COO\n        else:\n            raise ValueError(f\"Unknown data format {format}\")\n\n        def decode_arff(fh: Any) -&gt; dict:\n            decoder = arff.ArffDecoder()\n            return decoder.decode(fh, encode_nominal=True, return_type=return_type)  # type: ignore\n\n        if filepath.suffix.endswith(\".gz\"):\n            with gzip.open(filename) as zipfile:\n                return decode_arff(zipfile)\n        else:\n            with filepath.open(encoding=\"utf8\") as fh:\n                return decode_arff(fh)\n\n    def _parse_data_from_arff(  # noqa: C901, PLR0912, PLR0915\n        self,\n        arff_file_path: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Parse all required data from arff file.\n\n        Parameters\n        ----------\n        arff_file_path : str\n            Path to the file on disk.\n\n        Returns\n        -------\n        Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]\n            DataFrame or csr_matrix: dataset\n            List[bool]: List indicating which columns contain categorical variables.\n            List[str]: List of column names.\n        \"\"\"\n        try:\n            data = self._get_arff(self.format)\n        except OSError as e:\n            logger.critical(\n                f\"Please check that the data file {arff_file_path} is \" \"there and can be read.\",\n            )\n            raise e\n\n        ARFF_DTYPES_TO_PD_DTYPE = {\n            \"INTEGER\": \"integer\",\n            \"REAL\": \"floating\",\n            \"NUMERIC\": \"floating\",\n            \"STRING\": \"string\",\n        }\n        attribute_dtype = {}\n        attribute_names = []\n        categories_names = {}\n        categorical = []\n        for name, type_ in data[\"attributes\"]:\n            # if the feature is nominal and a sparse matrix is\n            # requested, the categories need to be numeric\n            if isinstance(type_, list) and self.format.lower() == \"sparse_arff\":\n                try:\n                    # checks if the strings which should be the class labels\n                    # can be encoded into integers\n                    pd.factorize(type_)[0]\n                except ValueError as e:\n                    raise ValueError(\n                        \"Categorical data needs to be numeric when using sparse ARFF.\"\n                    ) from e\n\n            # string can only be supported with pandas DataFrame\n            elif type_ == \"STRING\" and self.format.lower() == \"sparse_arff\":\n                raise ValueError(\"Dataset containing strings is not supported with sparse ARFF.\")\n\n            # infer the dtype from the ARFF header\n            if isinstance(type_, list):\n                categorical.append(True)\n                categories_names[name] = type_\n                if len(type_) == 2:\n                    type_norm = [cat.lower().capitalize() for cat in type_]\n                    if {\"True\", \"False\"} == set(type_norm):\n                        categories_names[name] = [cat == \"True\" for cat in type_norm]\n                        attribute_dtype[name] = \"boolean\"\n                    else:\n                        attribute_dtype[name] = \"categorical\"\n                else:\n                    attribute_dtype[name] = \"categorical\"\n            else:\n                categorical.append(False)\n                attribute_dtype[name] = ARFF_DTYPES_TO_PD_DTYPE[type_]\n            attribute_names.append(name)\n\n        if self.format.lower() == \"sparse_arff\":\n            X = data[\"data\"]\n            X_shape = (max(X[1]) + 1, max(X[2]) + 1)\n            X = scipy.sparse.coo_matrix((X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32)\n            X = X.tocsr()\n        elif self.format.lower() == \"arff\":\n            X = pd.DataFrame(data[\"data\"], columns=attribute_names)\n\n            col = []\n            for column_name in X.columns:\n                if attribute_dtype[column_name] in (\"categorical\", \"boolean\"):\n                    categories = self._unpack_categories(\n                        X[column_name],  # type: ignore\n                        categories_names[column_name],\n                    )\n                    col.append(categories)\n                elif attribute_dtype[column_name] in (\"floating\", \"integer\"):\n                    X_col = X[column_name]\n                    if X_col.min() &gt;= 0 and X_col.max() &lt;= 255:\n                        try:\n                            X_col_uint = X_col.astype(\"uint8\")\n                            if (X_col == X_col_uint).all():\n                                col.append(X_col_uint)\n                                continue\n                        except ValueError:\n                            pass\n                    col.append(X[column_name])\n                else:\n                    col.append(X[column_name])\n            X = pd.concat(col, axis=1)\n        else:\n            raise ValueError(f\"Dataset format '{self.format}' is not a valid format.\")\n\n        return X, categorical, attribute_names  # type: ignore\n\n    def _compressed_cache_file_paths(self, data_file: Path) -&gt; tuple[Path, Path, Path]:\n        data_pickle_file = data_file.with_suffix(\".pkl.py3\")\n        data_feather_file = data_file.with_suffix(\".feather\")\n        feather_attribute_file = data_file.with_suffix(\".feather.attributes.pkl.py3\")\n        return data_pickle_file, data_feather_file, feather_attribute_file\n\n    def _cache_compressed_file_from_file(\n        self,\n        data_file: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Store data from the local file in compressed format.\n\n        If a local parquet file is present it will be used instead of the arff file.\n        Sets cache_format to 'pickle' if data is sparse.\n        \"\"\"\n        (\n            data_pickle_file,\n            data_feather_file,\n            feather_attribute_file,\n        ) = self._compressed_cache_file_paths(data_file)\n\n        if data_file.suffix == \".arff\":\n            data, categorical, attribute_names = self._parse_data_from_arff(data_file)\n        elif data_file.suffix == \".pq\":\n            try:\n                data = pd.read_parquet(data_file)\n            except Exception as e:  # noqa: BLE001\n                raise Exception(f\"File: {data_file}\") from e\n\n            categorical = [data[c].dtype.name == \"category\" for c in data.columns]\n            attribute_names = list(data.columns)\n        else:\n            raise ValueError(f\"Unknown file type for file '{data_file}'.\")\n\n        # Feather format does not work for sparse datasets, so we use pickle for sparse datasets\n        if scipy.sparse.issparse(data):\n            self.cache_format = \"pickle\"\n\n        logger.info(f\"{self.cache_format} write {self.name}\")\n        if self.cache_format == \"feather\":\n            assert isinstance(data, pd.DataFrame)\n\n            data.to_feather(data_feather_file)\n            with open(feather_attribute_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_feather_file = data_feather_file\n            self.feather_attribute_file = feather_attribute_file\n\n        else:\n            with open(data_pickle_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_pickle_file = data_pickle_file\n\n        data_file = data_pickle_file if self.cache_format == \"pickle\" else data_feather_file\n        logger.debug(f\"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}\")\n\n        return data, categorical, attribute_names\n\n    def _load_data(self) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:  # noqa: PLR0912, C901\n        \"\"\"Load data from compressed format or arff. Download data if not present on disk.\"\"\"\n        need_to_create_pickle = self.cache_format == \"pickle\" and self.data_pickle_file is None\n        need_to_create_feather = self.cache_format == \"feather\" and self.data_feather_file is None\n\n        if need_to_create_pickle or need_to_create_feather:\n            if self.data_file is None:\n                self._download_data()\n\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n\n        # helper variable to help identify where errors occur\n        fpath = self.data_feather_file if self.cache_format == \"feather\" else self.data_pickle_file\n        logger.info(f\"{self.cache_format} load data {self.name}\")\n        try:\n            if self.cache_format == \"feather\":\n                assert self.data_feather_file is not None\n                assert self.feather_attribute_file is not None\n\n                data = pd.read_feather(self.data_feather_file)\n                fpath = self.feather_attribute_file\n                with open(self.feather_attribute_file, \"rb\") as fh:  # noqa: PTH123\n                    categorical, attribute_names = pickle.load(fh)  # noqa: S301\n            else:\n                assert self.data_pickle_file is not None\n                with open(self.data_pickle_file, \"rb\") as fh:  # noqa: PTH123\n                    data, categorical, attribute_names = pickle.load(fh)  # noqa: S301\n        except FileNotFoundError as e:\n            raise ValueError(\n                f\"Cannot find file for dataset {self.name} at location '{fpath}'.\"\n            ) from e\n        except (EOFError, ModuleNotFoundError, ValueError, AttributeError) as e:\n            error_message = getattr(e, \"message\", e.args[0])\n            hint = \"\"\n\n            if isinstance(e, EOFError):\n                readable_error = \"Detected a corrupt cache file\"\n            elif isinstance(e, (ModuleNotFoundError, AttributeError)):\n                readable_error = \"Detected likely dependency issues\"\n                hint = (\n                    \"This can happen if the cache was constructed with a different pandas version \"\n                    \"than the one that is used to load the data. See also \"\n                )\n                if isinstance(e, ModuleNotFoundError):\n                    hint += \"https://github.com/openml/openml-python/issues/918. \"\n                elif isinstance(e, AttributeError):\n                    hint += \"https://github.com/openml/openml-python/pull/1121. \"\n\n            elif isinstance(e, ValueError) and \"unsupported pickle protocol\" in e.args[0]:\n                readable_error = \"Encountered unsupported pickle protocol\"\n            else:\n                raise e\n\n            logger.warning(\n                f\"{readable_error} when loading dataset {self.id} from '{fpath}'. \"\n                f\"{hint}\"\n                f\"Error message was: {error_message}. \"\n                \"We will continue loading data from the arff-file, \"\n                \"but this will be much slower for big datasets. \"\n                \"Please manually delete the cache file if you want OpenML-Python \"\n                \"to attempt to reconstruct it.\",\n            )\n            assert self.data_file is not None\n            data, categorical, attribute_names = self._parse_data_from_arff(Path(self.data_file))\n\n        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)\n        if self.cache_format == \"pickle\" and not data_up_to_date:\n            logger.info(\"Updating outdated pickle file.\")\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n        return data, categorical, attribute_names\n\n    # TODO(eddiebergman): Can type this better with overload\n    # TODO(eddiebergman): Could also techinically use scipy.sparse.sparray\n    @staticmethod\n    def _convert_array_format(\n        data: pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix,\n        array_format: Literal[\"array\", \"dataframe\"],\n        attribute_names: list | None = None,\n    ) -&gt; pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix:\n        \"\"\"Convert a dataset to a given array format.\n\n        Converts to numpy array if data is non-sparse.\n        Converts to a sparse dataframe if data is sparse.\n\n        Parameters\n        ----------\n        array_format : str {'array', 'dataframe'}\n            Desired data type of the output\n            - If array_format='array'\n                If data is non-sparse\n                    Converts to numpy-array\n                    Enforces numeric encoding of categorical columns\n                    Missing values are represented as NaN in the numpy-array\n                else returns data as is\n            - If array_format='dataframe'\n                If data is sparse\n                    Works only on sparse data\n                    Converts sparse data to sparse dataframe\n                else returns data as is\n\n        \"\"\"\n        if array_format == \"array\" and not isinstance(data, scipy.sparse.spmatrix):\n            # We encode the categories such that they are integer to be able\n            # to make a conversion to numeric for backward compatibility\n            def _encode_if_category(column: pd.Series | np.ndarray) -&gt; pd.Series | np.ndarray:\n                if column.dtype.name == \"category\":\n                    column = column.cat.codes.astype(np.float32)\n                    mask_nan = column == -1\n                    column[mask_nan] = np.nan\n                return column\n\n            if isinstance(data, pd.DataFrame):\n                columns = {\n                    column_name: _encode_if_category(data.loc[:, column_name])\n                    for column_name in data.columns\n                }\n                data = pd.DataFrame(columns)\n            else:\n                data = _encode_if_category(data)\n\n            try:\n                # TODO(eddiebergman): float32?\n                return_array = np.asarray(data, dtype=np.float32)\n            except ValueError as e:\n                raise PyOpenMLError(\n                    \"PyOpenML cannot handle string when returning numpy\"\n                    ' arrays. Use dataset_format=\"dataframe\".',\n                ) from e\n\n            return return_array\n\n        if array_format == \"dataframe\":\n            if scipy.sparse.issparse(data):\n                data = pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)\n        else:\n            data_type = \"sparse-data\" if scipy.sparse.issparse(data) else \"non-sparse data\"\n            logger.warning(\n                f\"Cannot convert {data_type} ({type(data)}) to '{array_format}'.\"\n                \" Returning input data.\",\n            )\n        return data\n\n    @staticmethod\n    def _unpack_categories(series: pd.Series, categories: list) -&gt; pd.Series:\n        # nan-likes can not be explicitly specified as a category\n        def valid_category(cat: Any) -&gt; bool:\n            return isinstance(cat, str) or (cat is not None and not np.isnan(cat))\n\n        filtered_categories = [c for c in categories if valid_category(c)]\n        col = []\n        for x in series:\n            try:\n                col.append(categories[int(x)])\n            except (TypeError, ValueError):\n                col.append(np.nan)\n\n        # We require two lines to create a series of categories as detailed here:\n        # https://pandas.pydata.org/pandas-docs/version/0.24/user_guide/categorical.html#series-creation\n        raw_cat = pd.Categorical(col, ordered=True, categories=filtered_categories)\n        return pd.Series(raw_cat, index=series.index, name=series.name)\n\n    def get_data(  # noqa: C901, PLR0912, PLR0915\n        self,\n        target: list[str] | str | None = None,\n        include_row_id: bool = False,  # noqa: FBT001, FBT002\n        include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n        np.ndarray | pd.DataFrame | None,\n        list[bool],\n        list[str],\n    ]:\n        \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n        Parameters\n        ----------\n        target : string, List[str] or None (default=None)\n            Name of target column to separate from the data.\n            Splitting multiple columns is currently not supported.\n        include_row_id : boolean (default=False)\n            Whether to include row ids in the returned dataset.\n        include_ignore_attribute : boolean (default=False)\n            Whether to include columns that are marked as \"ignore\"\n            on the server in the dataset.\n        dataset_format : string (default='dataframe')\n            The format of returned dataset.\n            If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n            matrix. Support for ``array`` will be removed in 0.15.\n            If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n        Returns\n        -------\n        X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n            Dataset\n        y : ndarray or pd.Series, shape (n_samples, ) or None\n            Target column\n        categorical_indicator : boolean ndarray\n            Mask that indicate categorical features.\n        attribute_names : List[str]\n            List of attribute names.\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        data, categorical, attribute_names = self._load_data()\n\n        to_exclude = []\n        if not include_row_id and self.row_id_attribute is not None:\n            if isinstance(self.row_id_attribute, str):\n                to_exclude.append(self.row_id_attribute)\n            elif isinstance(self.row_id_attribute, Iterable):\n                to_exclude.extend(self.row_id_attribute)\n\n        if not include_ignore_attribute and self.ignore_attribute is not None:\n            if isinstance(self.ignore_attribute, str):\n                to_exclude.append(self.ignore_attribute)\n            elif isinstance(self.ignore_attribute, Iterable):\n                to_exclude.extend(self.ignore_attribute)\n\n        if len(to_exclude) &gt; 0:\n            logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n            keep = np.array([column not in to_exclude for column in attribute_names])\n            data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n            categorical = [cat for cat, k in zip(categorical, keep) if k]\n            attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n        if target is None:\n            data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n            targets = None\n        else:\n            if isinstance(target, str):\n                target = target.split(\",\") if \",\" in target else [target]\n            targets = np.array([column in target for column in attribute_names])\n            target_names = [column for column in attribute_names if column in target]\n            if np.sum(targets) &gt; 1:\n                raise NotImplementedError(\n                    \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n                )\n            target_categorical = [\n                cat for cat, column in zip(categorical, attribute_names) if column in target\n            ]\n            target_dtype = int if target_categorical[0] else float\n\n            if isinstance(data, pd.DataFrame):\n                x = data.iloc[:, ~targets]\n                y = data.iloc[:, targets]\n            else:\n                x = data[:, ~targets]\n                y = data[:, targets].astype(target_dtype)  # type: ignore\n\n            categorical = [cat for cat, t in zip(categorical, targets) if not t]\n            attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n            x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n            if dataset_format == \"array\" and scipy.sparse.issparse(y):\n                # scikit-learn requires dense representation of targets\n                y = np.asarray(y.todense()).astype(target_dtype)\n                # dense representation of single column sparse arrays become a 2-d array\n                # need to flatten it to a 1-d array for _convert_array_format()\n                y = y.squeeze()\n            y = self._convert_array_format(y, dataset_format, target_names)\n            y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n            if len(y.shape) &gt; 1 and y.shape[1] == 1:\n                # single column targets should be 1-d for both `array` and `dataframe` formats\n                assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n                y = y.squeeze()\n            data, targets = x, y\n\n        return data, targets, categorical, attribute_names  # type: ignore\n\n    def _load_features(self) -&gt; None:\n        \"\"\"Load the features metadata from the server and store it in the dataset object.\"\"\"\n        # Delayed Import to avoid circular imports or having to import all of dataset.functions to\n        # import OpenMLDataset.\n        from openml.datasets.functions import _get_dataset_features_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        features_file = _get_dataset_features_file(None, self.dataset_id)\n        self._features = _read_features(features_file)\n\n    def _load_qualities(self) -&gt; None:\n        \"\"\"Load qualities information from the server and store it in the dataset object.\"\"\"\n        # same reason as above for _load_features\n        from openml.datasets.functions import _get_dataset_qualities_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        qualities_file = _get_dataset_qualities_file(None, self.dataset_id)\n\n        if qualities_file is None:\n            self._no_qualities_found = True\n        else:\n            self._qualities = _read_qualities(qualities_file)\n\n    def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n        \"\"\"Reads the datasets arff to determine the class-labels.\n\n        If the task has no class labels (for example a regression problem)\n        it returns None. Necessary because the data returned by get_data\n        only contains the indices of the classes, while OpenML needs the real\n        classname when uploading the results of a run.\n\n        Parameters\n        ----------\n        target_name : str\n            Name of the target attribute\n\n        Returns\n        -------\n        list\n        \"\"\"\n        for feature in self.features.values():\n            if feature.name == target_name:\n                if feature.data_type == \"nominal\":\n                    return feature.nominal_values\n\n                if feature.data_type == \"string\":\n                    # Rel.: #1311\n                    # The target is invalid for a classification task if the feature type is string\n                    # and not nominal. For such miss-configured tasks, we silently fix it here as\n                    # we can safely interpreter string as nominal.\n                    df, *_ = self.get_data()\n                    return list(df[feature.name].unique())\n\n        return None\n\n    def get_features_by_type(  # noqa: C901\n        self,\n        data_type: str,\n        exclude: list[str] | None = None,\n        exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n        exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; list[int]:\n        \"\"\"\n        Return indices of features of a given type, e.g. all nominal features.\n        Optional parameters to exclude various features by index or ontology.\n\n        Parameters\n        ----------\n        data_type : str\n            The data type to return (e.g., nominal, numeric, date, string)\n        exclude : list(int)\n            List of columns to exclude from the return value\n        exclude_ignore_attribute : bool\n            Whether to exclude the defined ignore attributes (and adapt the\n            return values as if these indices are not present)\n        exclude_row_id_attribute : bool\n            Whether to exclude the defined row id attributes (and adapt the\n            return values as if these indices are not present)\n\n        Returns\n        -------\n        result : list\n            a list of indices that have the specified data type\n        \"\"\"\n        if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n            raise TypeError(\"Illegal feature type requested\")\n        if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n            raise TypeError(\"ignore_attribute should be a list\")\n        if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n            raise TypeError(\"row id attribute should be a str\")\n        if exclude is not None and not isinstance(exclude, list):\n            raise TypeError(\"Exclude should be a list\")\n            # assert all(isinstance(elem, str) for elem in exclude),\n            #            \"Exclude should be a list of strings\"\n        to_exclude = []\n        if exclude is not None:\n            to_exclude.extend(exclude)\n        if exclude_ignore_attribute and self.ignore_attribute is not None:\n            to_exclude.extend(self.ignore_attribute)\n        if exclude_row_id_attribute and self.row_id_attribute is not None:\n            to_exclude.append(self.row_id_attribute)\n\n        result = []\n        offset = 0\n        # this function assumes that everything in to_exclude will\n        # be 'excluded' from the dataset (hence the offset)\n        for idx in self.features:\n            name = self.features[idx].name\n            if name in to_exclude:\n                offset += 1\n            elif self.features[idx].data_type == data_type:\n                result.append(idx - offset)\n        return result\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Adds the 'dataset' to file elements.\"\"\"\n        file_elements: dict = {}\n        path = None if self.data_file is None else Path(self.data_file).absolute()\n\n        if self._dataset is not None:\n            file_elements[\"dataset\"] = self._dataset\n        elif path is not None and path.exists():\n            with path.open(\"rb\") as fp:\n                file_elements[\"dataset\"] = fp.read()\n\n            try:\n                dataset_utf8 = str(file_elements[\"dataset\"], encoding=\"utf8\")\n                arff.ArffDecoder().decode(dataset_utf8, encode_nominal=True)\n            except arff.ArffException as e:\n                raise ValueError(\"The file you have provided is not a valid arff file.\") from e\n\n        elif self.url is None:\n            raise ValueError(\"No valid url/path to the data file was given.\")\n        return file_elements\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.dataset_id = int(xml_response[\"oml:upload_data_set\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        props = [\n            \"id\",\n            \"name\",\n            \"version\",\n            \"description\",\n            \"format\",\n            \"creator\",\n            \"contributor\",\n            \"collection_date\",\n            \"upload_date\",\n            \"language\",\n            \"licence\",\n            \"url\",\n            \"default_target_attribute\",\n            \"row_id_attribute\",\n            \"ignore_attribute\",\n            \"version_label\",\n            \"citation\",\n            \"tag\",\n            \"visibility\",\n            \"original_data_url\",\n            \"paper_url\",\n            \"update_comment\",\n            \"md5_checksum\",\n        ]\n\n        prop_values = {}\n        for prop in props:\n            content = getattr(self, prop, None)\n            if content is not None:\n                prop_values[\"oml:\" + prop] = content\n\n        return {\n            \"oml:data_set_description\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **prop_values,\n            }\n        }\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.features","title":"<code>features: dict[int, OpenMLDataFeature]</code>  <code>property</code>","text":"<p>Get the features of this dataset.</p>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Get the dataset numeric id.</p>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.qualities","title":"<code>qualities: dict[str, float] | None</code>  <code>property</code>","text":"<p>Get the qualities of this dataset.</p>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.get_data","title":"<code>get_data(target=None, include_row_id=False, include_ignore_attribute=False, dataset_format='dataframe')</code>","text":"<p>Returns dataset content as dataframes or sparse matrices.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>(string, List[str] or None(default=None))</code> <p>Name of target column to separate from the data. Splitting multiple columns is currently not supported.</p> <code>None</code> <code>include_row_id</code> <code>boolean(default=False)</code> <p>Whether to include row ids in the returned dataset.</p> <code>False</code> <code>include_ignore_attribute</code> <code>boolean(default=False)</code> <p>Whether to include columns that are marked as \"ignore\" on the server in the dataset.</p> <code>False</code> <code>dataset_format</code> <code>string(default='dataframe')</code> <p>The format of returned dataset. If <code>array</code>, the returned dataset will be a NumPy array or a SciPy sparse matrix. Support for <code>array</code> will be removed in 0.15. If <code>dataframe</code>, the returned dataset will be a Pandas DataFrame.</p> <code>'dataframe'</code> <p>Returns:</p> Name Type Description <code>X</code> <code>ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)</code> <p>Dataset</p> <code>y</code> <code>(ndarray or Series, shape(n_samples) or None)</code> <p>Target column</p> <code>categorical_indicator</code> <code>boolean ndarray</code> <p>Mask that indicate categorical features.</p> <code>attribute_names</code> <code>List[str]</code> <p>List of attribute names.</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_data(  # noqa: C901, PLR0912, PLR0915\n    self,\n    target: list[str] | str | None = None,\n    include_row_id: bool = False,  # noqa: FBT001, FBT002\n    include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n    np.ndarray | pd.DataFrame | None,\n    list[bool],\n    list[str],\n]:\n    \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n    Parameters\n    ----------\n    target : string, List[str] or None (default=None)\n        Name of target column to separate from the data.\n        Splitting multiple columns is currently not supported.\n    include_row_id : boolean (default=False)\n        Whether to include row ids in the returned dataset.\n    include_ignore_attribute : boolean (default=False)\n        Whether to include columns that are marked as \"ignore\"\n        on the server in the dataset.\n    dataset_format : string (default='dataframe')\n        The format of returned dataset.\n        If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n        matrix. Support for ``array`` will be removed in 0.15.\n        If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n    Returns\n    -------\n    X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n        Dataset\n    y : ndarray or pd.Series, shape (n_samples, ) or None\n        Target column\n    categorical_indicator : boolean ndarray\n        Mask that indicate categorical features.\n    attribute_names : List[str]\n        List of attribute names.\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    data, categorical, attribute_names = self._load_data()\n\n    to_exclude = []\n    if not include_row_id and self.row_id_attribute is not None:\n        if isinstance(self.row_id_attribute, str):\n            to_exclude.append(self.row_id_attribute)\n        elif isinstance(self.row_id_attribute, Iterable):\n            to_exclude.extend(self.row_id_attribute)\n\n    if not include_ignore_attribute and self.ignore_attribute is not None:\n        if isinstance(self.ignore_attribute, str):\n            to_exclude.append(self.ignore_attribute)\n        elif isinstance(self.ignore_attribute, Iterable):\n            to_exclude.extend(self.ignore_attribute)\n\n    if len(to_exclude) &gt; 0:\n        logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n        keep = np.array([column not in to_exclude for column in attribute_names])\n        data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n        categorical = [cat for cat, k in zip(categorical, keep) if k]\n        attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n    if target is None:\n        data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n        targets = None\n    else:\n        if isinstance(target, str):\n            target = target.split(\",\") if \",\" in target else [target]\n        targets = np.array([column in target for column in attribute_names])\n        target_names = [column for column in attribute_names if column in target]\n        if np.sum(targets) &gt; 1:\n            raise NotImplementedError(\n                \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n            )\n        target_categorical = [\n            cat for cat, column in zip(categorical, attribute_names) if column in target\n        ]\n        target_dtype = int if target_categorical[0] else float\n\n        if isinstance(data, pd.DataFrame):\n            x = data.iloc[:, ~targets]\n            y = data.iloc[:, targets]\n        else:\n            x = data[:, ~targets]\n            y = data[:, targets].astype(target_dtype)  # type: ignore\n\n        categorical = [cat for cat, t in zip(categorical, targets) if not t]\n        attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n        x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n        if dataset_format == \"array\" and scipy.sparse.issparse(y):\n            # scikit-learn requires dense representation of targets\n            y = np.asarray(y.todense()).astype(target_dtype)\n            # dense representation of single column sparse arrays become a 2-d array\n            # need to flatten it to a 1-d array for _convert_array_format()\n            y = y.squeeze()\n        y = self._convert_array_format(y, dataset_format, target_names)\n        y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n        if len(y.shape) &gt; 1 and y.shape[1] == 1:\n            # single column targets should be 1-d for both `array` and `dataframe` formats\n            assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n            y = y.squeeze()\n        data, targets = x, y\n\n    return data, targets, categorical, attribute_names  # type: ignore\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.get_features_by_type","title":"<code>get_features_by_type(data_type, exclude=None, exclude_ignore_attribute=True, exclude_row_id_attribute=True)</code>","text":"<p>Return indices of features of a given type, e.g. all nominal features. Optional parameters to exclude various features by index or ontology.</p> <p>Parameters:</p> Name Type Description Default <code>data_type</code> <code>str</code> <p>The data type to return (e.g., nominal, numeric, date, string)</p> required <code>exclude</code> <code>list(int)</code> <p>List of columns to exclude from the return value</p> <code>None</code> <code>exclude_ignore_attribute</code> <code>bool</code> <p>Whether to exclude the defined ignore attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <code>exclude_row_id_attribute</code> <code>bool</code> <p>Whether to exclude the defined row id attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <p>Returns:</p> Name Type Description <code>result</code> <code>list</code> <p>a list of indices that have the specified data type</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_features_by_type(  # noqa: C901\n    self,\n    data_type: str,\n    exclude: list[str] | None = None,\n    exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n    exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n) -&gt; list[int]:\n    \"\"\"\n    Return indices of features of a given type, e.g. all nominal features.\n    Optional parameters to exclude various features by index or ontology.\n\n    Parameters\n    ----------\n    data_type : str\n        The data type to return (e.g., nominal, numeric, date, string)\n    exclude : list(int)\n        List of columns to exclude from the return value\n    exclude_ignore_attribute : bool\n        Whether to exclude the defined ignore attributes (and adapt the\n        return values as if these indices are not present)\n    exclude_row_id_attribute : bool\n        Whether to exclude the defined row id attributes (and adapt the\n        return values as if these indices are not present)\n\n    Returns\n    -------\n    result : list\n        a list of indices that have the specified data type\n    \"\"\"\n    if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n        raise TypeError(\"Illegal feature type requested\")\n    if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n        raise TypeError(\"ignore_attribute should be a list\")\n    if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n        raise TypeError(\"row id attribute should be a str\")\n    if exclude is not None and not isinstance(exclude, list):\n        raise TypeError(\"Exclude should be a list\")\n        # assert all(isinstance(elem, str) for elem in exclude),\n        #            \"Exclude should be a list of strings\"\n    to_exclude = []\n    if exclude is not None:\n        to_exclude.extend(exclude)\n    if exclude_ignore_attribute and self.ignore_attribute is not None:\n        to_exclude.extend(self.ignore_attribute)\n    if exclude_row_id_attribute and self.row_id_attribute is not None:\n        to_exclude.append(self.row_id_attribute)\n\n    result = []\n    offset = 0\n    # this function assumes that everything in to_exclude will\n    # be 'excluded' from the dataset (hence the offset)\n    for idx in self.features:\n        name = self.features[idx].name\n        if name in to_exclude:\n            offset += 1\n        elif self.features[idx].data_type == data_type:\n            result.append(idx - offset)\n    return result\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.retrieve_class_labels","title":"<code>retrieve_class_labels(target_name='class')</code>","text":"<p>Reads the datasets arff to determine the class-labels.</p> <p>If the task has no class labels (for example a regression problem) it returns None. Necessary because the data returned by get_data only contains the indices of the classes, while OpenML needs the real classname when uploading the results of a run.</p> <p>Parameters:</p> Name Type Description Default <code>target_name</code> <code>str</code> <p>Name of the target attribute</p> <code>'class'</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n    \"\"\"Reads the datasets arff to determine the class-labels.\n\n    If the task has no class labels (for example a regression problem)\n    it returns None. Necessary because the data returned by get_data\n    only contains the indices of the classes, while OpenML needs the real\n    classname when uploading the results of a run.\n\n    Parameters\n    ----------\n    target_name : str\n        Name of the target attribute\n\n    Returns\n    -------\n    list\n    \"\"\"\n    for feature in self.features.values():\n        if feature.name == target_name:\n            if feature.data_type == \"nominal\":\n                return feature.nominal_values\n\n            if feature.data_type == \"string\":\n                # Rel.: #1311\n                # The target is invalid for a classification task if the feature type is string\n                # and not nominal. For such miss-configured tasks, we silently fix it here as\n                # we can safely interpreter string as nominal.\n                df, *_ = self.get_data()\n                return list(df[feature.name].unique())\n\n    return None\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.attributes_arff_from_df","title":"<code>attributes_arff_from_df(df)</code>","text":"<p>Describe attributes of the dataframe according to ARFF specification.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>(DataFrame, shape(n_samples, n_features))</code> <p>The dataframe containing the data set.</p> required <p>Returns:</p> Name Type Description <code>attributes_arff</code> <code>list[str]</code> <p>The data set attributes as required by the ARFF format.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def attributes_arff_from_df(df: pd.DataFrame) -&gt; list[tuple[str, list[str] | str]]:\n    \"\"\"Describe attributes of the dataframe according to ARFF specification.\n\n    Parameters\n    ----------\n    df : DataFrame, shape (n_samples, n_features)\n        The dataframe containing the data set.\n\n    Returns\n    -------\n    attributes_arff : list[str]\n        The data set attributes as required by the ARFF format.\n    \"\"\"\n    PD_DTYPES_TO_ARFF_DTYPE = {\"integer\": \"INTEGER\", \"floating\": \"REAL\", \"string\": \"STRING\"}\n    attributes_arff: list[tuple[str, list[str] | str]] = []\n\n    if not all(isinstance(column_name, str) for column_name in df.columns):\n        logger.warning(\"Converting non-str column names to str.\")\n        df.columns = [str(column_name) for column_name in df.columns]\n\n    for column_name in df:\n        # skipna=True does not infer properly the dtype. The NA values are\n        # dropped before the inference instead.\n        column_dtype = pd.api.types.infer_dtype(df[column_name].dropna(), skipna=False)\n\n        if column_dtype == \"categorical\":\n            # for categorical feature, arff expects a list string. However, a\n            # categorical column can contain mixed type and should therefore\n            # raise an error asking to convert all entries to string.\n            categories = df[column_name].cat.categories\n            categories_dtype = pd.api.types.infer_dtype(categories)\n            if categories_dtype not in (\"string\", \"unicode\"):\n                raise ValueError(\n                    f\"The column '{column_name}' of the dataframe is of \"\n                    \"'category' dtype. Therefore, all values in \"\n                    \"this columns should be string. Please \"\n                    \"convert the entries which are not string. \"\n                    f\"Got {categories_dtype} dtype in this column.\",\n                )\n            attributes_arff.append((column_name, categories.tolist()))\n        elif column_dtype == \"boolean\":\n            # boolean are encoded as categorical.\n            attributes_arff.append((column_name, [\"True\", \"False\"]))\n        elif column_dtype in PD_DTYPES_TO_ARFF_DTYPE:\n            attributes_arff.append((column_name, PD_DTYPES_TO_ARFF_DTYPE[column_dtype]))\n        else:\n            raise ValueError(\n                f\"The dtype '{column_dtype}' of the column '{column_name}' is not \"\n                \"currently supported by liac-arff. Supported \"\n                \"dtypes are categorical, string, integer, \"\n                \"floating, and boolean.\",\n            )\n    return attributes_arff\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.check_datasets_active","title":"<code>check_datasets_active(dataset_ids, raise_error_if_not_exist=True)</code>","text":"<p>Check if the dataset ids provided are active.</p> <p>Raises an error if a dataset_id in the given list of dataset_ids does not exist on the server and <code>raise_error_if_not_exist</code> is set to True (default).</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>List[int]</code> <p>A list of integers representing dataset ids.</p> required <code>raise_error_if_not_exist</code> <code>bool(default=True)</code> <p>Flag that if activated can raise an error, if one or more of the given dataset ids do not exist on the server.</p> <code>True</code> <p>Returns:</p> Type Description <code>dict</code> <p>A dictionary with items {did: bool}</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def check_datasets_active(\n    dataset_ids: list[int],\n    raise_error_if_not_exist: bool = True,  # noqa: FBT001, FBT002\n) -&gt; dict[int, bool]:\n    \"\"\"\n    Check if the dataset ids provided are active.\n\n    Raises an error if a dataset_id in the given list\n    of dataset_ids does not exist on the server and\n    `raise_error_if_not_exist` is set to True (default).\n\n    Parameters\n    ----------\n    dataset_ids : List[int]\n        A list of integers representing dataset ids.\n    raise_error_if_not_exist : bool (default=True)\n        Flag that if activated can raise an error, if one or more of the\n        given dataset ids do not exist on the server.\n\n    Returns\n    -------\n    dict\n        A dictionary with items {did: bool}\n    \"\"\"\n    datasets = list_datasets(status=\"all\", data_id=dataset_ids, output_format=\"dataframe\")\n    missing = set(dataset_ids) - set(datasets.get(\"did\", []))\n    if raise_error_if_not_exist and missing:\n        missing_str = \", \".join(str(did) for did in missing)\n        raise ValueError(f\"Could not find dataset(s) {missing_str} in OpenML dataset list.\")\n    return dict(datasets[\"status\"] == \"active\")\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.create_dataset","title":"<code>create_dataset(name, description, creator, contributor, collection_date, language, licence, attributes, data, default_target_attribute, ignore_attribute, citation, row_id_attribute=None, original_data_url=None, paper_url=None, update_comment=None, version_label=None)</code>","text":"<p>Create a dataset.</p> <p>This function creates an OpenMLDataset object. The OpenMLDataset object contains information related to the dataset and the actual data file.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> required <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> required <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> required <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> required <code>licence</code> <code>str</code> <p>License of the data.</p> required <code>attributes</code> <code>list, dict, or 'auto'</code> <p>A list of tuples. Each tuple consists of the attribute name and type. If passing a pandas DataFrame, the attributes can be automatically inferred by passing <code>'auto'</code>. Specific attributes can be manually specified by a passing a dictionary where the key is the name of the attribute and the value is the data type of the attribute.</p> required <code>data</code> <code>(ndarray, list, dataframe, coo_matrix, shape(n_samples, n_features))</code> <p>An array that contains both the attributes and the targets. When providing a dataframe, the attribute names and type can be inferred by passing <code>attributes='auto'</code>. The target feature is indicated as meta-data of the dataset.</p> required <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> required <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes. Can have multiple values, comma separated.</p> required <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> required <code>version_label</code> <code>str</code> <p>Version label provided by user.  Can be a date, hash, or some other type of id.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>class</code> <code>`openml.OpenMLDataset`</code> <code>Dataset description.</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def create_dataset(  # noqa: C901, PLR0912, PLR0915\n    name: str,\n    description: str | None,\n    creator: str | None,\n    contributor: str | None,\n    collection_date: str | None,\n    language: str | None,\n    licence: str | None,\n    # TODO(eddiebergman): Docstring says `type` but I don't know what this is other than strings\n    # Edit: Found it could also be like [\"True\", \"False\"]\n    attributes: list[tuple[str, str | list[str]]] | dict[str, str | list[str]] | Literal[\"auto\"],\n    data: pd.DataFrame | np.ndarray | scipy.sparse.coo_matrix,\n    # TODO(eddiebergman): Function requires `default_target_attribute` exist but API allows None\n    default_target_attribute: str,\n    ignore_attribute: str | list[str] | None,\n    citation: str,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n    update_comment: str | None = None,\n    version_label: str | None = None,\n) -&gt; OpenMLDataset:\n    \"\"\"Create a dataset.\n\n    This function creates an OpenMLDataset object.\n    The OpenMLDataset object contains information related to the dataset\n    and the actual data file.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str\n        License of the data.\n    attributes : list, dict, or 'auto'\n        A list of tuples. Each tuple consists of the attribute name and type.\n        If passing a pandas DataFrame, the attributes can be automatically\n        inferred by passing ``'auto'``. Specific attributes can be manually\n        specified by a passing a dictionary where the key is the name of the\n        attribute and the value is the data type of the attribute.\n    data : ndarray, list, dataframe, coo_matrix, shape (n_samples, n_features)\n        An array that contains both the attributes and the targets. When\n        providing a dataframe, the attribute names and type can be inferred by\n        passing ``attributes='auto'``.\n        The target feature is indicated as meta-data of the dataset.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n        Can have multiple values, comma separated.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    version_label : str, optional\n        Version label provided by user.\n         Can be a date, hash, or some other type of id.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n\n    Returns\n    -------\n    class:`openml.OpenMLDataset`\n    Dataset description.\n    \"\"\"\n    if isinstance(data, pd.DataFrame):\n        # infer the row id from the index of the dataset\n        if row_id_attribute is None:\n            row_id_attribute = data.index.name\n        # When calling data.values, the index will be skipped.\n        # We need to reset the index such that it is part of the data.\n        if data.index.name is not None:\n            data = data.reset_index()\n\n    if attributes == \"auto\" or isinstance(attributes, dict):\n        if not isinstance(data, pd.DataFrame):\n            raise ValueError(\n                \"Automatically inferring attributes requires \"\n                f\"a pandas DataFrame. A {data!r} was given instead.\",\n            )\n        # infer the type of data for each column of the DataFrame\n        attributes_ = attributes_arff_from_df(data)\n        if isinstance(attributes, dict):\n            # override the attributes which was specified by the user\n            for attr_idx in range(len(attributes_)):\n                attr_name = attributes_[attr_idx][0]\n                if attr_name in attributes:\n                    attributes_[attr_idx] = (attr_name, attributes[attr_name])\n    else:\n        attributes_ = attributes\n    ignore_attributes = _expand_parameter(ignore_attribute)\n    _validated_data_attributes(ignore_attributes, attributes_, \"ignore_attribute\")\n\n    default_target_attributes = _expand_parameter(default_target_attribute)\n    _validated_data_attributes(default_target_attributes, attributes_, \"default_target_attribute\")\n\n    if row_id_attribute is not None:\n        is_row_id_an_attribute = any(attr[0] == row_id_attribute for attr in attributes_)\n        if not is_row_id_an_attribute:\n            raise ValueError(\n                \"'row_id_attribute' should be one of the data attribute. \"\n                \" Got '{}' while candidates are {}.\".format(\n                    row_id_attribute,\n                    [attr[0] for attr in attributes_],\n                ),\n            )\n\n    if isinstance(data, pd.DataFrame):\n        if all(isinstance(dtype, pd.SparseDtype) for dtype in data.dtypes):\n            data = data.sparse.to_coo()\n            # liac-arff only support COO matrices with sorted rows\n            row_idx_sorted = np.argsort(data.row)  # type: ignore\n            data.row = data.row[row_idx_sorted]  # type: ignore\n            data.col = data.col[row_idx_sorted]  # type: ignore\n            data.data = data.data[row_idx_sorted]  # type: ignore\n        else:\n            data = data.to_numpy()\n\n    data_format: Literal[\"arff\", \"sparse_arff\"]\n    if isinstance(data, (list, np.ndarray)):\n        if isinstance(data[0], (list, np.ndarray)):\n            data_format = \"arff\"\n        elif isinstance(data[0], dict):\n            data_format = \"sparse_arff\"\n        else:\n            raise ValueError(\n                \"When giving a list or a numpy.ndarray, \"\n                \"they should contain a list/ numpy.ndarray \"\n                \"for dense data or a dictionary for sparse \"\n                f\"data. Got {data[0]!r} instead.\",\n            )\n    elif isinstance(data, coo_matrix):\n        data_format = \"sparse_arff\"\n    else:\n        raise ValueError(\n            \"When giving a list or a numpy.ndarray, \"\n            \"they should contain a list/ numpy.ndarray \"\n            \"for dense data or a dictionary for sparse \"\n            f\"data. Got {data[0]!r} instead.\",\n        )\n\n    arff_object = {\n        \"relation\": name,\n        \"description\": description,\n        \"attributes\": attributes_,\n        \"data\": data,\n    }\n\n    # serializes the ARFF dataset object and returns a string\n    arff_dataset = arff.dumps(arff_object)\n    try:\n        # check if ARFF is valid\n        decoder = arff.ArffDecoder()\n        return_type = arff.COO if data_format == \"sparse_arff\" else arff.DENSE\n        decoder.decode(arff_dataset, encode_nominal=True, return_type=return_type)\n    except arff.ArffException as e:\n        raise ValueError(\n            \"The arguments you have provided do not construct a valid ARFF file\"\n        ) from e\n\n    return OpenMLDataset(\n        name=name,\n        description=description,\n        data_format=data_format,\n        creator=creator,\n        contributor=contributor,\n        collection_date=collection_date,\n        language=language,\n        licence=licence,\n        default_target_attribute=default_target_attribute,\n        row_id_attribute=row_id_attribute,\n        ignore_attribute=ignore_attribute,\n        citation=citation,\n        version_label=version_label,\n        original_data_url=original_data_url,\n        paper_url=paper_url,\n        update_comment=update_comment,\n        dataset=arff_dataset,\n    )\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.delete_dataset","title":"<code>delete_dataset(dataset_id)</code>","text":"<p>Delete dataset with id <code>dataset_id</code> from the OpenML server.</p> <p>This can only be done if you are the owner of the dataset and no tasks are attached to the dataset.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int</code> <p>OpenML id of the dataset</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def delete_dataset(dataset_id: int) -&gt; bool:\n    \"\"\"Delete dataset with id `dataset_id` from the OpenML server.\n\n    This can only be done if you are the owner of the dataset and\n    no tasks are attached to the dataset.\n\n    Parameters\n    ----------\n    dataset_id : int\n        OpenML id of the dataset\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"data\", dataset_id)\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.edit_dataset","title":"<code>edit_dataset(data_id, description=None, creator=None, contributor=None, collection_date=None, language=None, default_target_attribute=None, ignore_attribute=None, citation=None, row_id_attribute=None, original_data_url=None, paper_url=None)</code>","text":"<p>Edits an OpenMLDataset.</p> <p>In addition to providing the dataset id of the dataset to edit (through data_id), you must specify a value for at least one of the optional function arguments, i.e. one value for a field to edit.</p> <p>This function allows editing of both non-critical and critical fields. Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.</p> <ul> <li>Editing non-critical data fields is allowed for all authenticated users.</li> <li>Editing critical fields is allowed only for the owner, provided there are no tasks    associated with this dataset.</li> </ul> <p>If dataset has tasks or if the user is not the owner, the only way to edit critical fields is to use fork_dataset followed by edit_dataset.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>ID of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dataset id</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def edit_dataset(\n    data_id: int,\n    description: str | None = None,\n    creator: str | None = None,\n    contributor: str | None = None,\n    collection_date: str | None = None,\n    language: str | None = None,\n    default_target_attribute: str | None = None,\n    ignore_attribute: str | list[str] | None = None,\n    citation: str | None = None,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n) -&gt; int:\n    \"\"\"Edits an OpenMLDataset.\n\n    In addition to providing the dataset id of the dataset to edit (through data_id),\n    you must specify a value for at least one of the optional function arguments,\n    i.e. one value for a field to edit.\n\n    This function allows editing of both non-critical and critical fields.\n    Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.\n\n     - Editing non-critical data fields is allowed for all authenticated users.\n     - Editing critical fields is allowed only for the owner, provided there are no tasks\n       associated with this dataset.\n\n    If dataset has tasks or if the user is not the owner, the only way\n    to edit critical fields is to use fork_dataset followed by edit_dataset.\n\n    Parameters\n    ----------\n    data_id : int\n        ID of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n\n    Returns\n    -------\n    Dataset id\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n\n    # compose data edit parameters as xml\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    xml = OrderedDict()  # type: 'OrderedDict[str, OrderedDict]'\n    xml[\"oml:data_edit_parameters\"] = OrderedDict()\n    xml[\"oml:data_edit_parameters\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n    xml[\"oml:data_edit_parameters\"][\"oml:description\"] = description\n    xml[\"oml:data_edit_parameters\"][\"oml:creator\"] = creator\n    xml[\"oml:data_edit_parameters\"][\"oml:contributor\"] = contributor\n    xml[\"oml:data_edit_parameters\"][\"oml:collection_date\"] = collection_date\n    xml[\"oml:data_edit_parameters\"][\"oml:language\"] = language\n    xml[\"oml:data_edit_parameters\"][\"oml:default_target_attribute\"] = default_target_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:row_id_attribute\"] = row_id_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:ignore_attribute\"] = ignore_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:citation\"] = citation\n    xml[\"oml:data_edit_parameters\"][\"oml:original_data_url\"] = original_data_url\n    xml[\"oml:data_edit_parameters\"][\"oml:paper_url\"] = paper_url\n\n    # delete None inputs\n    for k in list(xml[\"oml:data_edit_parameters\"]):\n        if not xml[\"oml:data_edit_parameters\"][k]:\n            del xml[\"oml:data_edit_parameters\"][k]\n\n    file_elements = {\n        \"edit_parameters\": (\"description.xml\", xmltodict.unparse(xml)),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        \"data/edit\",\n        \"post\",\n        data=form_data,\n        file_elements=file_elements,\n    )\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_edit\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.fork_dataset","title":"<code>fork_dataset(data_id)</code>","text":"<p>Creates a new dataset version, with the authenticated user as the new owner.  The forked dataset can have distinct dataset meta-data,  but the actual data itself is shared with the original version.</p> <p>This API is intended for use when a user is unable to edit the critical fields of a dataset  through the edit_dataset API.  (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)</p> <p>Specifically, this happens when the user is:         1. Not the owner of the dataset.         2. User is the owner of the dataset, but the dataset has tasks.</p> <p>In these two cases the only way to edit critical fields is:         1. STEP 1: Fork the dataset using fork_dataset API         2. STEP 2: Call edit_dataset API on the forked version.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to be forked</p> required <p>Returns:</p> Type Description <code>Dataset id of the forked dataset</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def fork_dataset(data_id: int) -&gt; int:\n    \"\"\"\n     Creates a new dataset version, with the authenticated user as the new owner.\n     The forked dataset can have distinct dataset meta-data,\n     but the actual data itself is shared with the original version.\n\n     This API is intended for use when a user is unable to edit the critical fields of a dataset\n     through the edit_dataset API.\n     (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)\n\n     Specifically, this happens when the user is:\n            1. Not the owner of the dataset.\n            2. User is the owner of the dataset, but the dataset has tasks.\n\n     In these two cases the only way to edit critical fields is:\n            1. STEP 1: Fork the dataset using fork_dataset API\n            2. STEP 2: Call edit_dataset API on the forked version.\n\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to be forked\n\n    Returns\n    -------\n    Dataset id of the forked dataset\n\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n    # compose data fork parameters\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"data/fork\", \"post\", data=form_data)\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_fork\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.get_dataset","title":"<code>get_dataset(dataset_id, download_data=None, version=None, error_if_multiple=False, cache_format='pickle', download_qualities=None, download_features_meta_data=None, download_all_files=False, force_refresh_cache=False)</code>","text":"<p>Download the OpenML dataset representation, optionally also download actual data file.</p> <p>This function is by default NOT thread/multiprocessing safe, as this function uses caching. A check will be performed to determine if the information has previously been downloaded to a cache, and if so be loaded from disk instead of retrieved from the server.</p> <p>To make this function thread safe, you can install the python package <code>oslo.concurrency</code>. If <code>oslo.concurrency</code> is installed <code>get_dataset</code> becomes thread safe.</p> <p>Alternatively, to make this function thread/multiprocessing safe initialize the cache first by calling <code>get_dataset(args)</code> once before calling <code>get_dataset(args)</code> many times in parallel. This will initialize the cache and later calls will use the cache in a thread/multiprocessing safe way.</p> <p>If dataset is retrieved by name, a version may be specified. If no version is specified and multiple versions of the dataset exist, the earliest version of the dataset that is still active will be returned. If no version is specified, multiple versions of the dataset exist and <code>exception_if_multiple</code> is set to <code>True</code>, this function will raise an exception.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int or str</code> <p>Dataset ID of the dataset to download</p> required <code>download_data</code> <code>bool(default=True)</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>None</code> <code>version</code> <code>(int, optional(default=None))</code> <p>Specifies the version if <code>dataset_id</code> is specified by name. If no version is specified, retrieve the least recent still active version.</p> <code>None</code> <code>error_if_multiple</code> <code>bool(default=False)</code> <p>If <code>True</code> raise an error if multiple datasets are found with matching criteria.</p> <code>False</code> <code>cache_format</code> <code>str(default='pickle') in {'pickle', 'feather'}</code> <p>Format for caching the dataset - may be feather or pickle Note that the default 'pickle' option may load slower than feather when no.of.rows is very high.</p> <code>'pickle'</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description. If True, download and cache the qualities file. If False, create the OpenMLDataset without qualities metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(qualities=True)</code> method.</p> <code>None</code> <code>download_features_meta_data</code> <code>bool(default=True)</code> <p>Option to download 'features' meta-data in addition to the minimal dataset description. If True, download and cache the features file. If False, create the OpenMLDataset without features metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(features=True)</code> method.</p> <code>None</code> <code>download_all_files</code> <code>bool</code> <p>EXPERIMENTAL. Download all files related to the dataset that reside on the server. Useful for datasets which refer to auxiliary files (e.g., meta-album).</p> <code>False</code> <code>force_refresh_cache</code> <code>bool(default=False)</code> <p>Force the cache to refreshed by deleting the cache directory and re-downloading the data. Note, if <code>force_refresh_cache</code> is True, <code>get_dataset</code> is NOT thread/multiprocessing safe, because this creates a race condition to creating and deleting the cache; as in general with the cache.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>dataset</code> <code>:class:`openml.OpenMLDataset`</code> <p>The downloaded dataset.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_dataset(  # noqa: C901, PLR0912\n    dataset_id: int | str,\n    download_data: bool | None = None,  # Optional for deprecation warning; later again only bool\n    version: int | None = None,\n    error_if_multiple: bool = False,  # noqa: FBT002, FBT001\n    cache_format: Literal[\"pickle\", \"feather\"] = \"pickle\",\n    download_qualities: bool | None = None,  # Same as above\n    download_features_meta_data: bool | None = None,  # Same as above\n    download_all_files: bool = False,  # noqa: FBT002, FBT001\n    force_refresh_cache: bool = False,  # noqa: FBT001, FBT002\n) -&gt; OpenMLDataset:\n    \"\"\"Download the OpenML dataset representation, optionally also download actual data file.\n\n    This function is by default NOT thread/multiprocessing safe, as this function uses caching.\n    A check will be performed to determine if the information has previously been downloaded to a\n    cache, and if so be loaded from disk instead of retrieved from the server.\n\n    To make this function thread safe, you can install the python package ``oslo.concurrency``.\n    If ``oslo.concurrency`` is installed `get_dataset` becomes thread safe.\n\n    Alternatively, to make this function thread/multiprocessing safe initialize the cache first by\n    calling `get_dataset(args)` once before calling `get_dataset(args)` many times in parallel.\n    This will initialize the cache and later calls will use the cache in a thread/multiprocessing\n    safe way.\n\n    If dataset is retrieved by name, a version may be specified.\n    If no version is specified and multiple versions of the dataset exist,\n    the earliest version of the dataset that is still active will be returned.\n    If no version is specified, multiple versions of the dataset exist and\n    ``exception_if_multiple`` is set to ``True``, this function will raise an exception.\n\n    Parameters\n    ----------\n    dataset_id : int or str\n        Dataset ID of the dataset to download\n    download_data : bool (default=True)\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    version : int, optional (default=None)\n        Specifies the version if `dataset_id` is specified by name.\n        If no version is specified, retrieve the least recent still active version.\n    error_if_multiple : bool (default=False)\n        If ``True`` raise an error if multiple datasets are found with matching criteria.\n    cache_format : str (default='pickle') in {'pickle', 'feather'}\n        Format for caching the dataset - may be feather or pickle\n        Note that the default 'pickle' option may load slower than feather when\n        no.of.rows is very high.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n        If True, download and cache the qualities file.\n        If False, create the OpenMLDataset without qualities metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(qualities=True)` method.\n    download_features_meta_data : bool (default=True)\n        Option to download 'features' meta-data in addition to the minimal dataset description.\n        If True, download and cache the features file.\n        If False, create the OpenMLDataset without features metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(features=True)` method.\n    download_all_files: bool (default=False)\n        EXPERIMENTAL. Download all files related to the dataset that reside on the server.\n        Useful for datasets which refer to auxiliary files (e.g., meta-album).\n    force_refresh_cache : bool (default=False)\n        Force the cache to refreshed by deleting the cache directory and re-downloading the data.\n        Note, if `force_refresh_cache` is True, `get_dataset` is NOT thread/multiprocessing safe,\n        because this creates a race condition to creating and deleting the cache; as in general with\n        the cache.\n\n    Returns\n    -------\n    dataset : :class:`openml.OpenMLDataset`\n        The downloaded dataset.\n    \"\"\"\n    # TODO(0.15): Remove the deprecation warning and make the default False; adjust types above\n    #   and documentation. Also remove None-to-True-cases below\n    if any(\n        download_flag is None\n        for download_flag in [download_data, download_qualities, download_features_meta_data]\n    ):\n        warnings.warn(\n            \"Starting from Version 0.15 `download_data`, `download_qualities`, and `download_featu\"\n            \"res_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy \"\n            \"loading. To disable this message until version 0.15 explicitly set `download_data`, \"\n            \"`download_qualities`, and `download_features_meta_data` to a bool while calling \"\n            \"`get_dataset`.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    download_data = True if download_data is None else download_data\n    download_qualities = True if download_qualities is None else download_qualities\n    download_features_meta_data = (\n        True if download_features_meta_data is None else download_features_meta_data\n    )\n\n    if download_all_files:\n        warnings.warn(\n            \"``download_all_files`` is experimental and is likely to break with new releases.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    if cache_format not in [\"feather\", \"pickle\"]:\n        raise ValueError(\n            \"cache_format must be one of 'feather' or 'pickle. \"\n            f\"Invalid format specified: {cache_format}\",\n        )\n\n    if isinstance(dataset_id, str):\n        try:\n            dataset_id = int(dataset_id)\n        except ValueError:\n            dataset_id = _name_to_id(dataset_id, version, error_if_multiple)  # type: ignore\n    elif not isinstance(dataset_id, int):\n        raise TypeError(\n            f\"`dataset_id` must be one of `str` or `int`, not {type(dataset_id)}.\",\n        )\n\n    if force_refresh_cache:\n        did_cache_dir = _get_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, dataset_id)\n        if did_cache_dir.exists():\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    did_cache_dir = _create_cache_directory_for_id(\n        DATASETS_CACHE_DIR_NAME,\n        dataset_id,\n    )\n\n    remove_dataset_cache = True\n    try:\n        description = _get_dataset_description(did_cache_dir, dataset_id)\n        features_file = None\n        qualities_file = None\n\n        if download_features_meta_data:\n            features_file = _get_dataset_features_file(did_cache_dir, dataset_id)\n        if download_qualities:\n            qualities_file = _get_dataset_qualities_file(did_cache_dir, dataset_id)\n\n        arff_file = _get_dataset_arff(description) if download_data else None\n        if \"oml:parquet_url\" in description and download_data:\n            try:\n                parquet_file = _get_dataset_parquet(\n                    description,\n                    download_all_files=download_all_files,\n                )\n            except urllib3.exceptions.MaxRetryError:\n                parquet_file = None\n            if parquet_file is None and arff_file:\n                logger.warning(\"Failed to download parquet, fallback on ARFF.\")\n        else:\n            parquet_file = None\n        remove_dataset_cache = False\n    except OpenMLServerException as e:\n        # if there was an exception\n        # check if the user had access to the dataset\n        if e.code == NO_ACCESS_GRANTED_ERRCODE:\n            raise OpenMLPrivateDatasetError(e.message) from None\n\n        raise e\n    finally:\n        if remove_dataset_cache:\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    return _create_dataset_from_description(\n        description,\n        features_file,\n        qualities_file,\n        arff_file,\n        parquet_file,\n        cache_format,\n    )\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.get_datasets","title":"<code>get_datasets(dataset_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download datasets.</p> <p>This function iterates :meth:<code>openml.datasets.get_dataset</code>.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>iterable</code> <p>Integers or strings representing dataset ids or dataset names. If dataset names are specified, the least recent still active dataset version is returned.</p> required <code>download_data</code> <code>bool</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>True</code> <code>download_qualities</code> <code>(bool, optional(default=True))</code> <p>If True, also download qualities.xml file. If False it skip the qualities.xml.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>list of datasets</code> <p>A list of dataset objects.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def get_datasets(\n    dataset_ids: list[str | int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLDataset]:\n    \"\"\"Download datasets.\n\n    This function iterates :meth:`openml.datasets.get_dataset`.\n\n    Parameters\n    ----------\n    dataset_ids : iterable\n        Integers or strings representing dataset ids or dataset names.\n        If dataset names are specified, the least recent still active dataset version is returned.\n    download_data : bool, optional\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    download_qualities : bool, optional (default=True)\n        If True, also download qualities.xml file. If False it skip the qualities.xml.\n\n    Returns\n    -------\n    datasets : list of datasets\n        A list of dataset objects.\n    \"\"\"\n    datasets = []\n    for dataset_id in dataset_ids:\n        datasets.append(\n            get_dataset(dataset_id, download_data, download_qualities=download_qualities),\n        )\n    return datasets\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.list_datasets","title":"<code>list_datasets(data_id=None, offset=None, size=None, status=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None, offset: int | None, size: int | None, status: str | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all dataset which are on OpenML. Supports large amount of results.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>list</code> <p>A list of data ids, to specify which datasets should be listed</p> <code>None</code> <code>offset</code> <code>int</code> <p>The number of datasets to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of datasets to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated}. By default active datasets are returned, but also datasets from another status can be requested.</p> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>output_format</code> <code>Literal['dataframe', 'dict']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators (keys in the dict): data_name, data_version, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from dataset ID to dict.</p> <p>Every dataset is represented by a dictionary containing the following information: - dataset id - name - format - status If qualities are calculated for the dataset, some of these are also returned.</p> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>dataset id</li> <li>name</li> <li>format</li> <li>status If qualities are calculated for the dataset, some of these are also included as columns.</li> </ul> </li> </ul> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_datasets(\n    data_id: list[int] | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dataframe\", \"dict\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all dataset which are on OpenML.\n    Supports large amount of results.\n\n    Parameters\n    ----------\n    data_id : list, optional\n        A list of data ids, to specify which datasets should be\n        listed\n    offset : int, optional\n        The number of datasets to skip, starting from the first.\n    size : int, optional\n        The maximum number of datasets to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated}. By\n        default active datasets are returned, but also datasets\n        from another status can be requested.\n    tag : str, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs : dict, optional\n        Legal filter operators (keys in the dict):\n        data_name, data_version, number_instances,\n        number_features, number_classes, number_missing_values.\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from dataset ID to dict.\n\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also included as columns.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        data_id=data_id,\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_datasets,\n        offset=offset,\n        size=size,\n        status=status,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.list_qualities","title":"<code>list_qualities()</code>","text":"<p>Return list of data qualities available.</p> <p>The function performs an API call to retrieve the entire list of data qualities that are computed on the datasets uploaded.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_qualities() -&gt; list[str]:\n    \"\"\"Return list of data qualities available.\n\n    The function performs an API call to retrieve the entire list of\n    data qualities that are computed on the datasets uploaded.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    api_call = \"data/qualities/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:quality\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:data_qualities_list\" not in qualities:\n        raise ValueError('Error in return XML, does not contain \"oml:data_qualities_list\"')\n\n    if not isinstance(qualities[\"oml:data_qualities_list\"][\"oml:quality\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:quality\" as a list')\n\n    return qualities[\"oml:data_qualities_list\"][\"oml:quality\"]\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.status_update","title":"<code>status_update(data_id, status)</code>","text":"<p>Updates the status of a dataset to either 'active' or 'deactivated'. Please see the OpenML API documentation for a description of the status and all legal status transitions: https://docs.openml.org/#dataset-status</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/datasets/functions.py</code> <pre><code>def status_update(data_id: int, status: Literal[\"active\", \"deactivated\"]) -&gt; None:\n    \"\"\"\n    Updates the status of a dataset to either 'active' or 'deactivated'.\n    Please see the OpenML API documentation for a description of the status\n    and all legal status transitions:\n    https://docs.openml.org/#dataset-status\n\n    Parameters\n    ----------\n    data_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(f\"Illegal status value. Legal values: {legal_status}\")\n\n    data: openml._api_calls.DATA_TYPE = {\"data_id\": data_id, \"status\": status}\n    result_xml = openml._api_calls._perform_api_call(\"data/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_data_id = result[\"oml:data_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:data_status_update\"][\"oml:status\"]\n    if status != server_status or int(data_id) != int(server_data_id):\n        # This should never happen\n        raise ValueError(\"Data id/status does not collide\")\n</code></pre>"},{"location":"reference/datasets/data_feature/","title":"data_feature","text":""},{"location":"reference/datasets/data_feature/#openml.datasets.data_feature.OpenMLDataFeature","title":"<code>OpenMLDataFeature</code>","text":"<p>Data Feature (a.k.a. Attribute) object.</p> <p>Parameters:</p> Name Type Description Default <code>index</code> <code>int</code> <p>The index of this feature</p> required <code>name</code> <code>str</code> <p>Name of the feature</p> required <code>data_type</code> <code>str</code> <p>can be nominal, numeric, string, date (corresponds to arff)</p> required <code>nominal_values</code> <code>list(str)</code> <p>list of the possible values, in case of nominal attribute</p> required <code>number_missing_values</code> <code>int</code> <p>Number of rows that have a missing value for this feature.</p> required <code>ontologies</code> <code>list(str)</code> <p>list of ontologies attached to this feature. An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided.</p> <code>None</code> Source code in <code>openml/datasets/data_feature.py</code> <pre><code>class OpenMLDataFeature:\n    \"\"\"\n    Data Feature (a.k.a. Attribute) object.\n\n    Parameters\n    ----------\n    index : int\n        The index of this feature\n    name : str\n        Name of the feature\n    data_type : str\n        can be nominal, numeric, string, date (corresponds to arff)\n    nominal_values : list(str)\n        list of the possible values, in case of nominal attribute\n    number_missing_values : int\n        Number of rows that have a missing value for this feature.\n    ontologies : list(str)\n        list of ontologies attached to this feature. An ontology describes the\n        concept that are described in a feature. An ontology is defined by an\n        URL where the information is provided.\n    \"\"\"\n\n    LEGAL_DATA_TYPES: ClassVar[Sequence[str]] = [\"nominal\", \"numeric\", \"string\", \"date\"]\n\n    def __init__(  # noqa: PLR0913\n        self,\n        index: int,\n        name: str,\n        data_type: str,\n        nominal_values: list[str],\n        number_missing_values: int,\n        ontologies: list[str] | None = None,\n    ):\n        if not isinstance(index, int):\n            raise TypeError(f\"Index must be `int` but is {type(index)}\")\n\n        if data_type not in self.LEGAL_DATA_TYPES:\n            raise ValueError(\n                f\"data type should be in {self.LEGAL_DATA_TYPES!s}, found: {data_type}\",\n            )\n\n        if data_type == \"nominal\":\n            if nominal_values is None:\n                raise TypeError(\n                    \"Dataset features require attribute `nominal_values` for nominal \"\n                    \"feature type.\",\n                )\n\n            if not isinstance(nominal_values, list):\n                raise TypeError(\n                    \"Argument `nominal_values` is of wrong datatype, should be list, \"\n                    f\"but is {type(nominal_values)}\",\n                )\n        elif nominal_values is not None:\n            raise TypeError(\"Argument `nominal_values` must be None for non-nominal feature.\")\n\n        if not isinstance(number_missing_values, int):\n            msg = f\"number_missing_values must be int but is {type(number_missing_values)}\"\n            raise TypeError(msg)\n\n        self.index = index\n        self.name = str(name)\n        self.data_type = str(data_type)\n        self.nominal_values = nominal_values\n        self.number_missing_values = number_missing_values\n        self.ontologies = ontologies\n\n    def __repr__(self) -&gt; str:\n        return \"[%d - %s (%s)]\" % (self.index, self.name, self.data_type)\n\n    def __eq__(self, other: Any) -&gt; bool:\n        return isinstance(other, OpenMLDataFeature) and self.__dict__ == other.__dict__\n\n    def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -&gt; None:  # noqa: FBT001, ARG002\n        pp.text(str(self))\n</code></pre>"},{"location":"reference/datasets/dataset/","title":"dataset","text":""},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset","title":"<code>OpenMLDataset</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>Dataset object.</p> <p>Allows fetching and uploading datasets to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>data_format</code> <code>str</code> <p>Format of the dataset which can be either 'arff' or 'sparse_arff'.</p> <code>'arff'</code> <code>cache_format</code> <code>str</code> <p>Format for caching the dataset which can be either 'feather' or 'pickle'.</p> <code>'pickle'</code> <code>dataset_id</code> <code>int</code> <p>Id autogenerated by the server.</p> <code>None</code> <code>version</code> <code>int</code> <p>Version of this dataset. '1' for original version. Auto-incremented by server.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>The date-time when the dataset was uploaded, generated by server.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>licence</code> <code>str</code> <p>License of the data.</p> <code>None</code> <code>url</code> <code>str</code> <p>Valid URL, points to actual data file. The file can be on the OpenML server or another dataset repository.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>version_label</code> <code>str</code> <p>Version label provided by user. Can be a date, hash, or some other type of id.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>tag</code> <code>str</code> <p>Tags, describing the algorithms.</p> <code>None</code> <code>visibility</code> <code>str</code> <p>Who can see the dataset. Typical values: 'Everyone','All my friends','Only me'. Can also be any of the user's circles.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <code>md5_checksum</code> <code>str</code> <p>MD5 checksum to check if the dataset is downloaded without corruption.</p> <code>None</code> <code>data_file</code> <code>str</code> <p>Path to where the dataset is located.</p> <code>None</code> <code>features_file</code> <code>dict</code> <p>A dictionary of dataset features, which maps a feature index to a OpenMLDataFeature.</p> <code>None</code> <code>qualities_file</code> <code>dict</code> <p>A dictionary of dataset qualities, which maps a quality name to a quality value.</p> <code>None</code> <code>dataset</code> <code>str | None</code> <p>Serialized arff dataset string.</p> <code>None</code> <code>parquet_url</code> <code>str | None</code> <p>This is the URL to the storage location where the dataset files are hosted. This can be a MinIO bucket URL. If specified, the data will be accessed from this URL when reading the files.</p> <code>None</code> <code>parquet_file</code> <code>str | None</code> <p>Path to the local file.</p> <code>None</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>class OpenMLDataset(OpenMLBase):\n    \"\"\"Dataset object.\n\n    Allows fetching and uploading datasets to OpenML.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    data_format : str\n        Format of the dataset which can be either 'arff' or 'sparse_arff'.\n    cache_format : str\n        Format for caching the dataset which can be either 'feather' or 'pickle'.\n    dataset_id : int, optional\n        Id autogenerated by the server.\n    version : int, optional\n        Version of this dataset. '1' for original version.\n        Auto-incremented by server.\n    creator : str, optional\n        The person who created the dataset.\n    contributor : str, optional\n        People who contributed to the current version of the dataset.\n    collection_date : str, optional\n        The date the data was originally collected, given by the uploader.\n    upload_date : str, optional\n        The date-time when the dataset was uploaded, generated by server.\n    language : str, optional\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str, optional\n        License of the data.\n    url : str, optional\n        Valid URL, points to actual data file.\n        The file can be on the OpenML server or another dataset repository.\n    default_target_attribute : str, optional\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column,\n        if present in the dataset.\n    ignore_attribute : str | list, optional\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    version_label : str, optional\n        Version label provided by user.\n        Can be a date, hash, or some other type of id.\n    citation : str, optional\n        Reference(s) that should be cited when building on this data.\n    tag : str, optional\n        Tags, describing the algorithms.\n    visibility : str, optional\n        Who can see the dataset.\n        Typical values: 'Everyone','All my friends','Only me'.\n        Can also be any of the user's circles.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n    md5_checksum : str, optional\n        MD5 checksum to check if the dataset is downloaded without corruption.\n    data_file : str, optional\n        Path to where the dataset is located.\n    features_file : dict, optional\n        A dictionary of dataset features,\n        which maps a feature index to a OpenMLDataFeature.\n    qualities_file : dict, optional\n        A dictionary of dataset qualities,\n        which maps a quality name to a quality value.\n    dataset: string, optional\n        Serialized arff dataset string.\n    parquet_url: string, optional\n        This is the URL to the storage location where the dataset files are hosted.\n        This can be a MinIO bucket URL. If specified, the data will be accessed\n        from this URL when reading the files.\n    parquet_file: string, optional\n        Path to the local file.\n    \"\"\"\n\n    def __init__(  # noqa: C901, PLR0912, PLR0913, PLR0915\n        self,\n        name: str,\n        description: str | None,\n        data_format: Literal[\"arff\", \"sparse_arff\"] = \"arff\",\n        cache_format: Literal[\"feather\", \"pickle\"] = \"pickle\",\n        dataset_id: int | None = None,\n        version: int | None = None,\n        creator: str | None = None,\n        contributor: str | None = None,\n        collection_date: str | None = None,\n        upload_date: str | None = None,\n        language: str | None = None,\n        licence: str | None = None,\n        url: str | None = None,\n        default_target_attribute: str | None = None,\n        row_id_attribute: str | None = None,\n        ignore_attribute: str | list[str] | None = None,\n        version_label: str | None = None,\n        citation: str | None = None,\n        tag: str | None = None,\n        visibility: str | None = None,\n        original_data_url: str | None = None,\n        paper_url: str | None = None,\n        update_comment: str | None = None,\n        md5_checksum: str | None = None,\n        data_file: str | None = None,\n        features_file: str | None = None,\n        qualities_file: str | None = None,\n        dataset: str | None = None,\n        parquet_url: str | None = None,\n        parquet_file: str | None = None,\n    ):\n        if cache_format not in [\"feather\", \"pickle\"]:\n            raise ValueError(\n                \"cache_format must be one of 'feather' or 'pickle. \"\n                f\"Invalid format specified: {cache_format}\",\n            )\n\n        def find_invalid_characters(string: str, pattern: str) -&gt; str:\n            invalid_chars = set()\n            regex = re.compile(pattern)\n            for char in string:\n                if not regex.match(char):\n                    invalid_chars.add(char)\n            return \",\".join(\n                [f\"'{char}'\" if char != \"'\" else f'\"{char}\"' for char in invalid_chars],\n            )\n\n        if dataset_id is None:\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if description and not re.match(pattern, description):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(description, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in description: {description}\",\n                )\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if citation and not re.match(pattern, citation):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(citation, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in citation: {citation}\",\n                )\n            pattern = \"^[a-zA-Z0-9_\\\\-\\\\.\\\\(\\\\),]+$\"\n            if not re.match(pattern, name):\n                # regex given by server in error message\n                invalid_characters = find_invalid_characters(name, pattern)\n                raise ValueError(f\"Invalid symbols {invalid_characters} in name: {name}\")\n\n        self.ignore_attribute: list[str] | None = None\n        if isinstance(ignore_attribute, str):\n            self.ignore_attribute = [ignore_attribute]\n        elif isinstance(ignore_attribute, list) or ignore_attribute is None:\n            self.ignore_attribute = ignore_attribute\n        else:\n            raise ValueError(\"Wrong data type for ignore_attribute. Should be list.\")\n\n        # TODO add function to check if the name is casual_string128\n        # Attributes received by querying the RESTful API\n        self.dataset_id = int(dataset_id) if dataset_id is not None else None\n        self.name = name\n        self.version = int(version) if version is not None else None\n        self.description = description\n        self.cache_format = cache_format\n        # Has to be called format, otherwise there will be an XML upload error\n        self.format = data_format\n        self.creator = creator\n        self.contributor = contributor\n        self.collection_date = collection_date\n        self.upload_date = upload_date\n        self.language = language\n        self.licence = licence\n        self.url = url\n        self.default_target_attribute = default_target_attribute\n        self.row_id_attribute = row_id_attribute\n\n        self.version_label = version_label\n        self.citation = citation\n        self.tag = tag\n        self.visibility = visibility\n        self.original_data_url = original_data_url\n        self.paper_url = paper_url\n        self.update_comment = update_comment\n        self.md5_checksum = md5_checksum\n        self.data_file = data_file\n        self.parquet_file = parquet_file\n        self._dataset = dataset\n        self._parquet_url = parquet_url\n\n        self._features: dict[int, OpenMLDataFeature] | None = None\n        self._qualities: dict[str, float] | None = None\n        self._no_qualities_found = False\n\n        if features_file is not None:\n            self._features = _read_features(Path(features_file))\n\n        # \"\" was the old default value by `get_dataset` and maybe still used by some\n        if qualities_file == \"\":\n            # TODO(0.15): to switch to \"qualities_file is not None\" below and remove warning\n            warnings.warn(\n                \"Starting from Version 0.15 `qualities_file` must be None and not an empty string \"\n                \"to avoid reading the qualities from file. Set `qualities_file` to None to avoid \"\n                \"this warning.\",\n                FutureWarning,\n                stacklevel=2,\n            )\n            qualities_file = None\n\n        if qualities_file is not None:\n            self._qualities = _read_qualities(Path(qualities_file))\n\n        if data_file is not None:\n            data_pickle, data_feather, feather_attribute = self._compressed_cache_file_paths(\n                Path(data_file)\n            )\n            self.data_pickle_file = data_pickle if Path(data_pickle).exists() else None\n            self.data_feather_file = data_feather if Path(data_feather).exists() else None\n            self.feather_attribute_file = feather_attribute if Path(feather_attribute) else None\n        else:\n            self.data_pickle_file = None\n            self.data_feather_file = None\n            self.feather_attribute_file = None\n\n    @property\n    def features(self) -&gt; dict[int, OpenMLDataFeature]:\n        \"\"\"Get the features of this dataset.\"\"\"\n        if self._features is None:\n            # TODO(eddiebergman): These should return a value so we can set it to be not None\n            self._load_features()\n\n        assert self._features is not None\n        return self._features\n\n    @property\n    def qualities(self) -&gt; dict[str, float] | None:\n        \"\"\"Get the qualities of this dataset.\"\"\"\n        # TODO(eddiebergman): Better docstring, I don't know what qualities means\n\n        # We have to check `_no_qualities_found` as there might not be qualities for a dataset\n        if self._qualities is None and (not self._no_qualities_found):\n            self._load_qualities()\n\n        return self._qualities\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Get the dataset numeric id.\"\"\"\n        return self.dataset_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Obtain number of features in accordance with lazy loading.\n        n_features: int | None = None\n        if self._qualities is not None and self._qualities[\"NumberOfFeatures\"] is not None:\n            n_features = int(self._qualities[\"NumberOfFeatures\"])\n        elif self._features is not None:\n            n_features = len(self._features)\n\n        fields: dict[str, int | str | None] = {\n            \"Name\": self.name,\n            \"Version\": self.version,\n            \"Format\": self.format,\n            \"Licence\": self.licence,\n            \"Download URL\": self.url,\n            \"Data file\": str(self.data_file) if self.data_file is not None else None,\n            \"Pickle file\": (\n                str(self.data_pickle_file) if self.data_pickle_file is not None else None\n            ),\n            \"# of features\": n_features,\n        }\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.dataset_id is not None:\n            fields[\"OpenML URL\"] = self.openml_url\n        if self._qualities is not None and self._qualities[\"NumberOfInstances\"] is not None:\n            fields[\"# of instances\"] = int(self._qualities[\"NumberOfInstances\"])\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Name\",\n            \"Version\",\n            \"Format\",\n            \"Upload Date\",\n            \"Licence\",\n            \"Download URL\",\n            \"OpenML URL\",\n            \"Data File\",\n            \"Pickle File\",\n            \"# of features\",\n            \"# of instances\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if not isinstance(other, OpenMLDataset):\n            return False\n\n        server_fields = {\n            \"dataset_id\",\n            \"version\",\n            \"upload_date\",\n            \"url\",\n            \"dataset\",\n            \"data_file\",\n        }\n\n        # check that common keys and values are identical\n        self_keys = set(self.__dict__.keys()) - server_fields\n        other_keys = set(other.__dict__.keys()) - server_fields\n        return self_keys == other_keys and all(\n            self.__dict__[key] == other.__dict__[key] for key in self_keys\n        )\n\n    def _download_data(self) -&gt; None:\n        \"\"\"Download ARFF data file to standard cache directory. Set `self.data_file`.\"\"\"\n        # import required here to avoid circular import.\n        from .functions import _get_dataset_arff, _get_dataset_parquet\n\n        self.data_file = str(_get_dataset_arff(self))\n        if self._parquet_url is not None:\n            self.parquet_file = str(_get_dataset_parquet(self))\n\n    def _get_arff(self, format: str) -&gt; dict:  # noqa: A002\n        \"\"\"Read ARFF file and return decoded arff.\n\n        Reads the file referenced in self.data_file.\n\n        Parameters\n        ----------\n        format : str\n            Format of the ARFF file.\n            Must be one of 'arff' or 'sparse_arff' or a string that will be either of those\n            when converted to lower case.\n\n\n\n        Returns\n        -------\n        dict\n            Decoded arff.\n\n        \"\"\"\n        # TODO: add a partial read method which only returns the attribute\n        # headers of the corresponding .arff file!\n        import struct\n\n        filename = self.data_file\n        assert filename is not None\n        filepath = Path(filename)\n\n        bits = 8 * struct.calcsize(\"P\")\n\n        # Files can be considered too large on a 32-bit system,\n        # if it exceeds 120mb (slightly more than covtype dataset size)\n        # This number is somewhat arbitrary.\n        if bits != 64:\n            MB_120 = 120_000_000\n            file_size = filepath.stat().st_size\n            if file_size &gt; MB_120:\n                raise NotImplementedError(\n                    f\"File {filename} too big for {file_size}-bit system ({bits} bytes).\",\n                )\n\n        if format.lower() == \"arff\":\n            return_type = arff.DENSE\n        elif format.lower() == \"sparse_arff\":\n            return_type = arff.COO\n        else:\n            raise ValueError(f\"Unknown data format {format}\")\n\n        def decode_arff(fh: Any) -&gt; dict:\n            decoder = arff.ArffDecoder()\n            return decoder.decode(fh, encode_nominal=True, return_type=return_type)  # type: ignore\n\n        if filepath.suffix.endswith(\".gz\"):\n            with gzip.open(filename) as zipfile:\n                return decode_arff(zipfile)\n        else:\n            with filepath.open(encoding=\"utf8\") as fh:\n                return decode_arff(fh)\n\n    def _parse_data_from_arff(  # noqa: C901, PLR0912, PLR0915\n        self,\n        arff_file_path: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Parse all required data from arff file.\n\n        Parameters\n        ----------\n        arff_file_path : str\n            Path to the file on disk.\n\n        Returns\n        -------\n        Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]\n            DataFrame or csr_matrix: dataset\n            List[bool]: List indicating which columns contain categorical variables.\n            List[str]: List of column names.\n        \"\"\"\n        try:\n            data = self._get_arff(self.format)\n        except OSError as e:\n            logger.critical(\n                f\"Please check that the data file {arff_file_path} is \" \"there and can be read.\",\n            )\n            raise e\n\n        ARFF_DTYPES_TO_PD_DTYPE = {\n            \"INTEGER\": \"integer\",\n            \"REAL\": \"floating\",\n            \"NUMERIC\": \"floating\",\n            \"STRING\": \"string\",\n        }\n        attribute_dtype = {}\n        attribute_names = []\n        categories_names = {}\n        categorical = []\n        for name, type_ in data[\"attributes\"]:\n            # if the feature is nominal and a sparse matrix is\n            # requested, the categories need to be numeric\n            if isinstance(type_, list) and self.format.lower() == \"sparse_arff\":\n                try:\n                    # checks if the strings which should be the class labels\n                    # can be encoded into integers\n                    pd.factorize(type_)[0]\n                except ValueError as e:\n                    raise ValueError(\n                        \"Categorical data needs to be numeric when using sparse ARFF.\"\n                    ) from e\n\n            # string can only be supported with pandas DataFrame\n            elif type_ == \"STRING\" and self.format.lower() == \"sparse_arff\":\n                raise ValueError(\"Dataset containing strings is not supported with sparse ARFF.\")\n\n            # infer the dtype from the ARFF header\n            if isinstance(type_, list):\n                categorical.append(True)\n                categories_names[name] = type_\n                if len(type_) == 2:\n                    type_norm = [cat.lower().capitalize() for cat in type_]\n                    if {\"True\", \"False\"} == set(type_norm):\n                        categories_names[name] = [cat == \"True\" for cat in type_norm]\n                        attribute_dtype[name] = \"boolean\"\n                    else:\n                        attribute_dtype[name] = \"categorical\"\n                else:\n                    attribute_dtype[name] = \"categorical\"\n            else:\n                categorical.append(False)\n                attribute_dtype[name] = ARFF_DTYPES_TO_PD_DTYPE[type_]\n            attribute_names.append(name)\n\n        if self.format.lower() == \"sparse_arff\":\n            X = data[\"data\"]\n            X_shape = (max(X[1]) + 1, max(X[2]) + 1)\n            X = scipy.sparse.coo_matrix((X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32)\n            X = X.tocsr()\n        elif self.format.lower() == \"arff\":\n            X = pd.DataFrame(data[\"data\"], columns=attribute_names)\n\n            col = []\n            for column_name in X.columns:\n                if attribute_dtype[column_name] in (\"categorical\", \"boolean\"):\n                    categories = self._unpack_categories(\n                        X[column_name],  # type: ignore\n                        categories_names[column_name],\n                    )\n                    col.append(categories)\n                elif attribute_dtype[column_name] in (\"floating\", \"integer\"):\n                    X_col = X[column_name]\n                    if X_col.min() &gt;= 0 and X_col.max() &lt;= 255:\n                        try:\n                            X_col_uint = X_col.astype(\"uint8\")\n                            if (X_col == X_col_uint).all():\n                                col.append(X_col_uint)\n                                continue\n                        except ValueError:\n                            pass\n                    col.append(X[column_name])\n                else:\n                    col.append(X[column_name])\n            X = pd.concat(col, axis=1)\n        else:\n            raise ValueError(f\"Dataset format '{self.format}' is not a valid format.\")\n\n        return X, categorical, attribute_names  # type: ignore\n\n    def _compressed_cache_file_paths(self, data_file: Path) -&gt; tuple[Path, Path, Path]:\n        data_pickle_file = data_file.with_suffix(\".pkl.py3\")\n        data_feather_file = data_file.with_suffix(\".feather\")\n        feather_attribute_file = data_file.with_suffix(\".feather.attributes.pkl.py3\")\n        return data_pickle_file, data_feather_file, feather_attribute_file\n\n    def _cache_compressed_file_from_file(\n        self,\n        data_file: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Store data from the local file in compressed format.\n\n        If a local parquet file is present it will be used instead of the arff file.\n        Sets cache_format to 'pickle' if data is sparse.\n        \"\"\"\n        (\n            data_pickle_file,\n            data_feather_file,\n            feather_attribute_file,\n        ) = self._compressed_cache_file_paths(data_file)\n\n        if data_file.suffix == \".arff\":\n            data, categorical, attribute_names = self._parse_data_from_arff(data_file)\n        elif data_file.suffix == \".pq\":\n            try:\n                data = pd.read_parquet(data_file)\n            except Exception as e:  # noqa: BLE001\n                raise Exception(f\"File: {data_file}\") from e\n\n            categorical = [data[c].dtype.name == \"category\" for c in data.columns]\n            attribute_names = list(data.columns)\n        else:\n            raise ValueError(f\"Unknown file type for file '{data_file}'.\")\n\n        # Feather format does not work for sparse datasets, so we use pickle for sparse datasets\n        if scipy.sparse.issparse(data):\n            self.cache_format = \"pickle\"\n\n        logger.info(f\"{self.cache_format} write {self.name}\")\n        if self.cache_format == \"feather\":\n            assert isinstance(data, pd.DataFrame)\n\n            data.to_feather(data_feather_file)\n            with open(feather_attribute_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_feather_file = data_feather_file\n            self.feather_attribute_file = feather_attribute_file\n\n        else:\n            with open(data_pickle_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_pickle_file = data_pickle_file\n\n        data_file = data_pickle_file if self.cache_format == \"pickle\" else data_feather_file\n        logger.debug(f\"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}\")\n\n        return data, categorical, attribute_names\n\n    def _load_data(self) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:  # noqa: PLR0912, C901\n        \"\"\"Load data from compressed format or arff. Download data if not present on disk.\"\"\"\n        need_to_create_pickle = self.cache_format == \"pickle\" and self.data_pickle_file is None\n        need_to_create_feather = self.cache_format == \"feather\" and self.data_feather_file is None\n\n        if need_to_create_pickle or need_to_create_feather:\n            if self.data_file is None:\n                self._download_data()\n\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n\n        # helper variable to help identify where errors occur\n        fpath = self.data_feather_file if self.cache_format == \"feather\" else self.data_pickle_file\n        logger.info(f\"{self.cache_format} load data {self.name}\")\n        try:\n            if self.cache_format == \"feather\":\n                assert self.data_feather_file is not None\n                assert self.feather_attribute_file is not None\n\n                data = pd.read_feather(self.data_feather_file)\n                fpath = self.feather_attribute_file\n                with open(self.feather_attribute_file, \"rb\") as fh:  # noqa: PTH123\n                    categorical, attribute_names = pickle.load(fh)  # noqa: S301\n            else:\n                assert self.data_pickle_file is not None\n                with open(self.data_pickle_file, \"rb\") as fh:  # noqa: PTH123\n                    data, categorical, attribute_names = pickle.load(fh)  # noqa: S301\n        except FileNotFoundError as e:\n            raise ValueError(\n                f\"Cannot find file for dataset {self.name} at location '{fpath}'.\"\n            ) from e\n        except (EOFError, ModuleNotFoundError, ValueError, AttributeError) as e:\n            error_message = getattr(e, \"message\", e.args[0])\n            hint = \"\"\n\n            if isinstance(e, EOFError):\n                readable_error = \"Detected a corrupt cache file\"\n            elif isinstance(e, (ModuleNotFoundError, AttributeError)):\n                readable_error = \"Detected likely dependency issues\"\n                hint = (\n                    \"This can happen if the cache was constructed with a different pandas version \"\n                    \"than the one that is used to load the data. See also \"\n                )\n                if isinstance(e, ModuleNotFoundError):\n                    hint += \"https://github.com/openml/openml-python/issues/918. \"\n                elif isinstance(e, AttributeError):\n                    hint += \"https://github.com/openml/openml-python/pull/1121. \"\n\n            elif isinstance(e, ValueError) and \"unsupported pickle protocol\" in e.args[0]:\n                readable_error = \"Encountered unsupported pickle protocol\"\n            else:\n                raise e\n\n            logger.warning(\n                f\"{readable_error} when loading dataset {self.id} from '{fpath}'. \"\n                f\"{hint}\"\n                f\"Error message was: {error_message}. \"\n                \"We will continue loading data from the arff-file, \"\n                \"but this will be much slower for big datasets. \"\n                \"Please manually delete the cache file if you want OpenML-Python \"\n                \"to attempt to reconstruct it.\",\n            )\n            assert self.data_file is not None\n            data, categorical, attribute_names = self._parse_data_from_arff(Path(self.data_file))\n\n        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)\n        if self.cache_format == \"pickle\" and not data_up_to_date:\n            logger.info(\"Updating outdated pickle file.\")\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n        return data, categorical, attribute_names\n\n    # TODO(eddiebergman): Can type this better with overload\n    # TODO(eddiebergman): Could also techinically use scipy.sparse.sparray\n    @staticmethod\n    def _convert_array_format(\n        data: pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix,\n        array_format: Literal[\"array\", \"dataframe\"],\n        attribute_names: list | None = None,\n    ) -&gt; pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix:\n        \"\"\"Convert a dataset to a given array format.\n\n        Converts to numpy array if data is non-sparse.\n        Converts to a sparse dataframe if data is sparse.\n\n        Parameters\n        ----------\n        array_format : str {'array', 'dataframe'}\n            Desired data type of the output\n            - If array_format='array'\n                If data is non-sparse\n                    Converts to numpy-array\n                    Enforces numeric encoding of categorical columns\n                    Missing values are represented as NaN in the numpy-array\n                else returns data as is\n            - If array_format='dataframe'\n                If data is sparse\n                    Works only on sparse data\n                    Converts sparse data to sparse dataframe\n                else returns data as is\n\n        \"\"\"\n        if array_format == \"array\" and not isinstance(data, scipy.sparse.spmatrix):\n            # We encode the categories such that they are integer to be able\n            # to make a conversion to numeric for backward compatibility\n            def _encode_if_category(column: pd.Series | np.ndarray) -&gt; pd.Series | np.ndarray:\n                if column.dtype.name == \"category\":\n                    column = column.cat.codes.astype(np.float32)\n                    mask_nan = column == -1\n                    column[mask_nan] = np.nan\n                return column\n\n            if isinstance(data, pd.DataFrame):\n                columns = {\n                    column_name: _encode_if_category(data.loc[:, column_name])\n                    for column_name in data.columns\n                }\n                data = pd.DataFrame(columns)\n            else:\n                data = _encode_if_category(data)\n\n            try:\n                # TODO(eddiebergman): float32?\n                return_array = np.asarray(data, dtype=np.float32)\n            except ValueError as e:\n                raise PyOpenMLError(\n                    \"PyOpenML cannot handle string when returning numpy\"\n                    ' arrays. Use dataset_format=\"dataframe\".',\n                ) from e\n\n            return return_array\n\n        if array_format == \"dataframe\":\n            if scipy.sparse.issparse(data):\n                data = pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)\n        else:\n            data_type = \"sparse-data\" if scipy.sparse.issparse(data) else \"non-sparse data\"\n            logger.warning(\n                f\"Cannot convert {data_type} ({type(data)}) to '{array_format}'.\"\n                \" Returning input data.\",\n            )\n        return data\n\n    @staticmethod\n    def _unpack_categories(series: pd.Series, categories: list) -&gt; pd.Series:\n        # nan-likes can not be explicitly specified as a category\n        def valid_category(cat: Any) -&gt; bool:\n            return isinstance(cat, str) or (cat is not None and not np.isnan(cat))\n\n        filtered_categories = [c for c in categories if valid_category(c)]\n        col = []\n        for x in series:\n            try:\n                col.append(categories[int(x)])\n            except (TypeError, ValueError):\n                col.append(np.nan)\n\n        # We require two lines to create a series of categories as detailed here:\n        # https://pandas.pydata.org/pandas-docs/version/0.24/user_guide/categorical.html#series-creation\n        raw_cat = pd.Categorical(col, ordered=True, categories=filtered_categories)\n        return pd.Series(raw_cat, index=series.index, name=series.name)\n\n    def get_data(  # noqa: C901, PLR0912, PLR0915\n        self,\n        target: list[str] | str | None = None,\n        include_row_id: bool = False,  # noqa: FBT001, FBT002\n        include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n        np.ndarray | pd.DataFrame | None,\n        list[bool],\n        list[str],\n    ]:\n        \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n        Parameters\n        ----------\n        target : string, List[str] or None (default=None)\n            Name of target column to separate from the data.\n            Splitting multiple columns is currently not supported.\n        include_row_id : boolean (default=False)\n            Whether to include row ids in the returned dataset.\n        include_ignore_attribute : boolean (default=False)\n            Whether to include columns that are marked as \"ignore\"\n            on the server in the dataset.\n        dataset_format : string (default='dataframe')\n            The format of returned dataset.\n            If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n            matrix. Support for ``array`` will be removed in 0.15.\n            If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n        Returns\n        -------\n        X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n            Dataset\n        y : ndarray or pd.Series, shape (n_samples, ) or None\n            Target column\n        categorical_indicator : boolean ndarray\n            Mask that indicate categorical features.\n        attribute_names : List[str]\n            List of attribute names.\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        data, categorical, attribute_names = self._load_data()\n\n        to_exclude = []\n        if not include_row_id and self.row_id_attribute is not None:\n            if isinstance(self.row_id_attribute, str):\n                to_exclude.append(self.row_id_attribute)\n            elif isinstance(self.row_id_attribute, Iterable):\n                to_exclude.extend(self.row_id_attribute)\n\n        if not include_ignore_attribute and self.ignore_attribute is not None:\n            if isinstance(self.ignore_attribute, str):\n                to_exclude.append(self.ignore_attribute)\n            elif isinstance(self.ignore_attribute, Iterable):\n                to_exclude.extend(self.ignore_attribute)\n\n        if len(to_exclude) &gt; 0:\n            logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n            keep = np.array([column not in to_exclude for column in attribute_names])\n            data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n            categorical = [cat for cat, k in zip(categorical, keep) if k]\n            attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n        if target is None:\n            data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n            targets = None\n        else:\n            if isinstance(target, str):\n                target = target.split(\",\") if \",\" in target else [target]\n            targets = np.array([column in target for column in attribute_names])\n            target_names = [column for column in attribute_names if column in target]\n            if np.sum(targets) &gt; 1:\n                raise NotImplementedError(\n                    \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n                )\n            target_categorical = [\n                cat for cat, column in zip(categorical, attribute_names) if column in target\n            ]\n            target_dtype = int if target_categorical[0] else float\n\n            if isinstance(data, pd.DataFrame):\n                x = data.iloc[:, ~targets]\n                y = data.iloc[:, targets]\n            else:\n                x = data[:, ~targets]\n                y = data[:, targets].astype(target_dtype)  # type: ignore\n\n            categorical = [cat for cat, t in zip(categorical, targets) if not t]\n            attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n            x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n            if dataset_format == \"array\" and scipy.sparse.issparse(y):\n                # scikit-learn requires dense representation of targets\n                y = np.asarray(y.todense()).astype(target_dtype)\n                # dense representation of single column sparse arrays become a 2-d array\n                # need to flatten it to a 1-d array for _convert_array_format()\n                y = y.squeeze()\n            y = self._convert_array_format(y, dataset_format, target_names)\n            y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n            if len(y.shape) &gt; 1 and y.shape[1] == 1:\n                # single column targets should be 1-d for both `array` and `dataframe` formats\n                assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n                y = y.squeeze()\n            data, targets = x, y\n\n        return data, targets, categorical, attribute_names  # type: ignore\n\n    def _load_features(self) -&gt; None:\n        \"\"\"Load the features metadata from the server and store it in the dataset object.\"\"\"\n        # Delayed Import to avoid circular imports or having to import all of dataset.functions to\n        # import OpenMLDataset.\n        from openml.datasets.functions import _get_dataset_features_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        features_file = _get_dataset_features_file(None, self.dataset_id)\n        self._features = _read_features(features_file)\n\n    def _load_qualities(self) -&gt; None:\n        \"\"\"Load qualities information from the server and store it in the dataset object.\"\"\"\n        # same reason as above for _load_features\n        from openml.datasets.functions import _get_dataset_qualities_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        qualities_file = _get_dataset_qualities_file(None, self.dataset_id)\n\n        if qualities_file is None:\n            self._no_qualities_found = True\n        else:\n            self._qualities = _read_qualities(qualities_file)\n\n    def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n        \"\"\"Reads the datasets arff to determine the class-labels.\n\n        If the task has no class labels (for example a regression problem)\n        it returns None. Necessary because the data returned by get_data\n        only contains the indices of the classes, while OpenML needs the real\n        classname when uploading the results of a run.\n\n        Parameters\n        ----------\n        target_name : str\n            Name of the target attribute\n\n        Returns\n        -------\n        list\n        \"\"\"\n        for feature in self.features.values():\n            if feature.name == target_name:\n                if feature.data_type == \"nominal\":\n                    return feature.nominal_values\n\n                if feature.data_type == \"string\":\n                    # Rel.: #1311\n                    # The target is invalid for a classification task if the feature type is string\n                    # and not nominal. For such miss-configured tasks, we silently fix it here as\n                    # we can safely interpreter string as nominal.\n                    df, *_ = self.get_data()\n                    return list(df[feature.name].unique())\n\n        return None\n\n    def get_features_by_type(  # noqa: C901\n        self,\n        data_type: str,\n        exclude: list[str] | None = None,\n        exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n        exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; list[int]:\n        \"\"\"\n        Return indices of features of a given type, e.g. all nominal features.\n        Optional parameters to exclude various features by index or ontology.\n\n        Parameters\n        ----------\n        data_type : str\n            The data type to return (e.g., nominal, numeric, date, string)\n        exclude : list(int)\n            List of columns to exclude from the return value\n        exclude_ignore_attribute : bool\n            Whether to exclude the defined ignore attributes (and adapt the\n            return values as if these indices are not present)\n        exclude_row_id_attribute : bool\n            Whether to exclude the defined row id attributes (and adapt the\n            return values as if these indices are not present)\n\n        Returns\n        -------\n        result : list\n            a list of indices that have the specified data type\n        \"\"\"\n        if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n            raise TypeError(\"Illegal feature type requested\")\n        if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n            raise TypeError(\"ignore_attribute should be a list\")\n        if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n            raise TypeError(\"row id attribute should be a str\")\n        if exclude is not None and not isinstance(exclude, list):\n            raise TypeError(\"Exclude should be a list\")\n            # assert all(isinstance(elem, str) for elem in exclude),\n            #            \"Exclude should be a list of strings\"\n        to_exclude = []\n        if exclude is not None:\n            to_exclude.extend(exclude)\n        if exclude_ignore_attribute and self.ignore_attribute is not None:\n            to_exclude.extend(self.ignore_attribute)\n        if exclude_row_id_attribute and self.row_id_attribute is not None:\n            to_exclude.append(self.row_id_attribute)\n\n        result = []\n        offset = 0\n        # this function assumes that everything in to_exclude will\n        # be 'excluded' from the dataset (hence the offset)\n        for idx in self.features:\n            name = self.features[idx].name\n            if name in to_exclude:\n                offset += 1\n            elif self.features[idx].data_type == data_type:\n                result.append(idx - offset)\n        return result\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Adds the 'dataset' to file elements.\"\"\"\n        file_elements: dict = {}\n        path = None if self.data_file is None else Path(self.data_file).absolute()\n\n        if self._dataset is not None:\n            file_elements[\"dataset\"] = self._dataset\n        elif path is not None and path.exists():\n            with path.open(\"rb\") as fp:\n                file_elements[\"dataset\"] = fp.read()\n\n            try:\n                dataset_utf8 = str(file_elements[\"dataset\"], encoding=\"utf8\")\n                arff.ArffDecoder().decode(dataset_utf8, encode_nominal=True)\n            except arff.ArffException as e:\n                raise ValueError(\"The file you have provided is not a valid arff file.\") from e\n\n        elif self.url is None:\n            raise ValueError(\"No valid url/path to the data file was given.\")\n        return file_elements\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.dataset_id = int(xml_response[\"oml:upload_data_set\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        props = [\n            \"id\",\n            \"name\",\n            \"version\",\n            \"description\",\n            \"format\",\n            \"creator\",\n            \"contributor\",\n            \"collection_date\",\n            \"upload_date\",\n            \"language\",\n            \"licence\",\n            \"url\",\n            \"default_target_attribute\",\n            \"row_id_attribute\",\n            \"ignore_attribute\",\n            \"version_label\",\n            \"citation\",\n            \"tag\",\n            \"visibility\",\n            \"original_data_url\",\n            \"paper_url\",\n            \"update_comment\",\n            \"md5_checksum\",\n        ]\n\n        prop_values = {}\n        for prop in props:\n            content = getattr(self, prop, None)\n            if content is not None:\n                prop_values[\"oml:\" + prop] = content\n\n        return {\n            \"oml:data_set_description\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **prop_values,\n            }\n        }\n</code></pre>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.features","title":"<code>features: dict[int, OpenMLDataFeature]</code>  <code>property</code>","text":"<p>Get the features of this dataset.</p>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Get the dataset numeric id.</p>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.qualities","title":"<code>qualities: dict[str, float] | None</code>  <code>property</code>","text":"<p>Get the qualities of this dataset.</p>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.get_data","title":"<code>get_data(target=None, include_row_id=False, include_ignore_attribute=False, dataset_format='dataframe')</code>","text":"<p>Returns dataset content as dataframes or sparse matrices.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>(string, List[str] or None(default=None))</code> <p>Name of target column to separate from the data. Splitting multiple columns is currently not supported.</p> <code>None</code> <code>include_row_id</code> <code>boolean(default=False)</code> <p>Whether to include row ids in the returned dataset.</p> <code>False</code> <code>include_ignore_attribute</code> <code>boolean(default=False)</code> <p>Whether to include columns that are marked as \"ignore\" on the server in the dataset.</p> <code>False</code> <code>dataset_format</code> <code>string(default='dataframe')</code> <p>The format of returned dataset. If <code>array</code>, the returned dataset will be a NumPy array or a SciPy sparse matrix. Support for <code>array</code> will be removed in 0.15. If <code>dataframe</code>, the returned dataset will be a Pandas DataFrame.</p> <code>'dataframe'</code> <p>Returns:</p> Name Type Description <code>X</code> <code>ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)</code> <p>Dataset</p> <code>y</code> <code>(ndarray or Series, shape(n_samples) or None)</code> <p>Target column</p> <code>categorical_indicator</code> <code>boolean ndarray</code> <p>Mask that indicate categorical features.</p> <code>attribute_names</code> <code>List[str]</code> <p>List of attribute names.</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_data(  # noqa: C901, PLR0912, PLR0915\n    self,\n    target: list[str] | str | None = None,\n    include_row_id: bool = False,  # noqa: FBT001, FBT002\n    include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n    np.ndarray | pd.DataFrame | None,\n    list[bool],\n    list[str],\n]:\n    \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n    Parameters\n    ----------\n    target : string, List[str] or None (default=None)\n        Name of target column to separate from the data.\n        Splitting multiple columns is currently not supported.\n    include_row_id : boolean (default=False)\n        Whether to include row ids in the returned dataset.\n    include_ignore_attribute : boolean (default=False)\n        Whether to include columns that are marked as \"ignore\"\n        on the server in the dataset.\n    dataset_format : string (default='dataframe')\n        The format of returned dataset.\n        If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n        matrix. Support for ``array`` will be removed in 0.15.\n        If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n    Returns\n    -------\n    X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n        Dataset\n    y : ndarray or pd.Series, shape (n_samples, ) or None\n        Target column\n    categorical_indicator : boolean ndarray\n        Mask that indicate categorical features.\n    attribute_names : List[str]\n        List of attribute names.\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    data, categorical, attribute_names = self._load_data()\n\n    to_exclude = []\n    if not include_row_id and self.row_id_attribute is not None:\n        if isinstance(self.row_id_attribute, str):\n            to_exclude.append(self.row_id_attribute)\n        elif isinstance(self.row_id_attribute, Iterable):\n            to_exclude.extend(self.row_id_attribute)\n\n    if not include_ignore_attribute and self.ignore_attribute is not None:\n        if isinstance(self.ignore_attribute, str):\n            to_exclude.append(self.ignore_attribute)\n        elif isinstance(self.ignore_attribute, Iterable):\n            to_exclude.extend(self.ignore_attribute)\n\n    if len(to_exclude) &gt; 0:\n        logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n        keep = np.array([column not in to_exclude for column in attribute_names])\n        data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n        categorical = [cat for cat, k in zip(categorical, keep) if k]\n        attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n    if target is None:\n        data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n        targets = None\n    else:\n        if isinstance(target, str):\n            target = target.split(\",\") if \",\" in target else [target]\n        targets = np.array([column in target for column in attribute_names])\n        target_names = [column for column in attribute_names if column in target]\n        if np.sum(targets) &gt; 1:\n            raise NotImplementedError(\n                \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n            )\n        target_categorical = [\n            cat for cat, column in zip(categorical, attribute_names) if column in target\n        ]\n        target_dtype = int if target_categorical[0] else float\n\n        if isinstance(data, pd.DataFrame):\n            x = data.iloc[:, ~targets]\n            y = data.iloc[:, targets]\n        else:\n            x = data[:, ~targets]\n            y = data[:, targets].astype(target_dtype)  # type: ignore\n\n        categorical = [cat for cat, t in zip(categorical, targets) if not t]\n        attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n        x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n        if dataset_format == \"array\" and scipy.sparse.issparse(y):\n            # scikit-learn requires dense representation of targets\n            y = np.asarray(y.todense()).astype(target_dtype)\n            # dense representation of single column sparse arrays become a 2-d array\n            # need to flatten it to a 1-d array for _convert_array_format()\n            y = y.squeeze()\n        y = self._convert_array_format(y, dataset_format, target_names)\n        y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n        if len(y.shape) &gt; 1 and y.shape[1] == 1:\n            # single column targets should be 1-d for both `array` and `dataframe` formats\n            assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n            y = y.squeeze()\n        data, targets = x, y\n\n    return data, targets, categorical, attribute_names  # type: ignore\n</code></pre>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.get_features_by_type","title":"<code>get_features_by_type(data_type, exclude=None, exclude_ignore_attribute=True, exclude_row_id_attribute=True)</code>","text":"<p>Return indices of features of a given type, e.g. all nominal features. Optional parameters to exclude various features by index or ontology.</p> <p>Parameters:</p> Name Type Description Default <code>data_type</code> <code>str</code> <p>The data type to return (e.g., nominal, numeric, date, string)</p> required <code>exclude</code> <code>list(int)</code> <p>List of columns to exclude from the return value</p> <code>None</code> <code>exclude_ignore_attribute</code> <code>bool</code> <p>Whether to exclude the defined ignore attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <code>exclude_row_id_attribute</code> <code>bool</code> <p>Whether to exclude the defined row id attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <p>Returns:</p> Name Type Description <code>result</code> <code>list</code> <p>a list of indices that have the specified data type</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_features_by_type(  # noqa: C901\n    self,\n    data_type: str,\n    exclude: list[str] | None = None,\n    exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n    exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n) -&gt; list[int]:\n    \"\"\"\n    Return indices of features of a given type, e.g. all nominal features.\n    Optional parameters to exclude various features by index or ontology.\n\n    Parameters\n    ----------\n    data_type : str\n        The data type to return (e.g., nominal, numeric, date, string)\n    exclude : list(int)\n        List of columns to exclude from the return value\n    exclude_ignore_attribute : bool\n        Whether to exclude the defined ignore attributes (and adapt the\n        return values as if these indices are not present)\n    exclude_row_id_attribute : bool\n        Whether to exclude the defined row id attributes (and adapt the\n        return values as if these indices are not present)\n\n    Returns\n    -------\n    result : list\n        a list of indices that have the specified data type\n    \"\"\"\n    if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n        raise TypeError(\"Illegal feature type requested\")\n    if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n        raise TypeError(\"ignore_attribute should be a list\")\n    if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n        raise TypeError(\"row id attribute should be a str\")\n    if exclude is not None and not isinstance(exclude, list):\n        raise TypeError(\"Exclude should be a list\")\n        # assert all(isinstance(elem, str) for elem in exclude),\n        #            \"Exclude should be a list of strings\"\n    to_exclude = []\n    if exclude is not None:\n        to_exclude.extend(exclude)\n    if exclude_ignore_attribute and self.ignore_attribute is not None:\n        to_exclude.extend(self.ignore_attribute)\n    if exclude_row_id_attribute and self.row_id_attribute is not None:\n        to_exclude.append(self.row_id_attribute)\n\n    result = []\n    offset = 0\n    # this function assumes that everything in to_exclude will\n    # be 'excluded' from the dataset (hence the offset)\n    for idx in self.features:\n        name = self.features[idx].name\n        if name in to_exclude:\n            offset += 1\n        elif self.features[idx].data_type == data_type:\n            result.append(idx - offset)\n    return result\n</code></pre>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.retrieve_class_labels","title":"<code>retrieve_class_labels(target_name='class')</code>","text":"<p>Reads the datasets arff to determine the class-labels.</p> <p>If the task has no class labels (for example a regression problem) it returns None. Necessary because the data returned by get_data only contains the indices of the classes, while OpenML needs the real classname when uploading the results of a run.</p> <p>Parameters:</p> Name Type Description Default <code>target_name</code> <code>str</code> <p>Name of the target attribute</p> <code>'class'</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n    \"\"\"Reads the datasets arff to determine the class-labels.\n\n    If the task has no class labels (for example a regression problem)\n    it returns None. Necessary because the data returned by get_data\n    only contains the indices of the classes, while OpenML needs the real\n    classname when uploading the results of a run.\n\n    Parameters\n    ----------\n    target_name : str\n        Name of the target attribute\n\n    Returns\n    -------\n    list\n    \"\"\"\n    for feature in self.features.values():\n        if feature.name == target_name:\n            if feature.data_type == \"nominal\":\n                return feature.nominal_values\n\n            if feature.data_type == \"string\":\n                # Rel.: #1311\n                # The target is invalid for a classification task if the feature type is string\n                # and not nominal. For such miss-configured tasks, we silently fix it here as\n                # we can safely interpreter string as nominal.\n                df, *_ = self.get_data()\n                return list(df[feature.name].unique())\n\n    return None\n</code></pre>"},{"location":"reference/datasets/functions/","title":"functions","text":""},{"location":"reference/datasets/functions/#openml.datasets.functions.attributes_arff_from_df","title":"<code>attributes_arff_from_df(df)</code>","text":"<p>Describe attributes of the dataframe according to ARFF specification.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>(DataFrame, shape(n_samples, n_features))</code> <p>The dataframe containing the data set.</p> required <p>Returns:</p> Name Type Description <code>attributes_arff</code> <code>list[str]</code> <p>The data set attributes as required by the ARFF format.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def attributes_arff_from_df(df: pd.DataFrame) -&gt; list[tuple[str, list[str] | str]]:\n    \"\"\"Describe attributes of the dataframe according to ARFF specification.\n\n    Parameters\n    ----------\n    df : DataFrame, shape (n_samples, n_features)\n        The dataframe containing the data set.\n\n    Returns\n    -------\n    attributes_arff : list[str]\n        The data set attributes as required by the ARFF format.\n    \"\"\"\n    PD_DTYPES_TO_ARFF_DTYPE = {\"integer\": \"INTEGER\", \"floating\": \"REAL\", \"string\": \"STRING\"}\n    attributes_arff: list[tuple[str, list[str] | str]] = []\n\n    if not all(isinstance(column_name, str) for column_name in df.columns):\n        logger.warning(\"Converting non-str column names to str.\")\n        df.columns = [str(column_name) for column_name in df.columns]\n\n    for column_name in df:\n        # skipna=True does not infer properly the dtype. The NA values are\n        # dropped before the inference instead.\n        column_dtype = pd.api.types.infer_dtype(df[column_name].dropna(), skipna=False)\n\n        if column_dtype == \"categorical\":\n            # for categorical feature, arff expects a list string. However, a\n            # categorical column can contain mixed type and should therefore\n            # raise an error asking to convert all entries to string.\n            categories = df[column_name].cat.categories\n            categories_dtype = pd.api.types.infer_dtype(categories)\n            if categories_dtype not in (\"string\", \"unicode\"):\n                raise ValueError(\n                    f\"The column '{column_name}' of the dataframe is of \"\n                    \"'category' dtype. Therefore, all values in \"\n                    \"this columns should be string. Please \"\n                    \"convert the entries which are not string. \"\n                    f\"Got {categories_dtype} dtype in this column.\",\n                )\n            attributes_arff.append((column_name, categories.tolist()))\n        elif column_dtype == \"boolean\":\n            # boolean are encoded as categorical.\n            attributes_arff.append((column_name, [\"True\", \"False\"]))\n        elif column_dtype in PD_DTYPES_TO_ARFF_DTYPE:\n            attributes_arff.append((column_name, PD_DTYPES_TO_ARFF_DTYPE[column_dtype]))\n        else:\n            raise ValueError(\n                f\"The dtype '{column_dtype}' of the column '{column_name}' is not \"\n                \"currently supported by liac-arff. Supported \"\n                \"dtypes are categorical, string, integer, \"\n                \"floating, and boolean.\",\n            )\n    return attributes_arff\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.check_datasets_active","title":"<code>check_datasets_active(dataset_ids, raise_error_if_not_exist=True)</code>","text":"<p>Check if the dataset ids provided are active.</p> <p>Raises an error if a dataset_id in the given list of dataset_ids does not exist on the server and <code>raise_error_if_not_exist</code> is set to True (default).</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>List[int]</code> <p>A list of integers representing dataset ids.</p> required <code>raise_error_if_not_exist</code> <code>bool(default=True)</code> <p>Flag that if activated can raise an error, if one or more of the given dataset ids do not exist on the server.</p> <code>True</code> <p>Returns:</p> Type Description <code>dict</code> <p>A dictionary with items {did: bool}</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def check_datasets_active(\n    dataset_ids: list[int],\n    raise_error_if_not_exist: bool = True,  # noqa: FBT001, FBT002\n) -&gt; dict[int, bool]:\n    \"\"\"\n    Check if the dataset ids provided are active.\n\n    Raises an error if a dataset_id in the given list\n    of dataset_ids does not exist on the server and\n    `raise_error_if_not_exist` is set to True (default).\n\n    Parameters\n    ----------\n    dataset_ids : List[int]\n        A list of integers representing dataset ids.\n    raise_error_if_not_exist : bool (default=True)\n        Flag that if activated can raise an error, if one or more of the\n        given dataset ids do not exist on the server.\n\n    Returns\n    -------\n    dict\n        A dictionary with items {did: bool}\n    \"\"\"\n    datasets = list_datasets(status=\"all\", data_id=dataset_ids, output_format=\"dataframe\")\n    missing = set(dataset_ids) - set(datasets.get(\"did\", []))\n    if raise_error_if_not_exist and missing:\n        missing_str = \", \".join(str(did) for did in missing)\n        raise ValueError(f\"Could not find dataset(s) {missing_str} in OpenML dataset list.\")\n    return dict(datasets[\"status\"] == \"active\")\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.create_dataset","title":"<code>create_dataset(name, description, creator, contributor, collection_date, language, licence, attributes, data, default_target_attribute, ignore_attribute, citation, row_id_attribute=None, original_data_url=None, paper_url=None, update_comment=None, version_label=None)</code>","text":"<p>Create a dataset.</p> <p>This function creates an OpenMLDataset object. The OpenMLDataset object contains information related to the dataset and the actual data file.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> required <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> required <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> required <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> required <code>licence</code> <code>str</code> <p>License of the data.</p> required <code>attributes</code> <code>list, dict, or 'auto'</code> <p>A list of tuples. Each tuple consists of the attribute name and type. If passing a pandas DataFrame, the attributes can be automatically inferred by passing <code>'auto'</code>. Specific attributes can be manually specified by a passing a dictionary where the key is the name of the attribute and the value is the data type of the attribute.</p> required <code>data</code> <code>(ndarray, list, dataframe, coo_matrix, shape(n_samples, n_features))</code> <p>An array that contains both the attributes and the targets. When providing a dataframe, the attribute names and type can be inferred by passing <code>attributes='auto'</code>. The target feature is indicated as meta-data of the dataset.</p> required <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> required <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes. Can have multiple values, comma separated.</p> required <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> required <code>version_label</code> <code>str</code> <p>Version label provided by user.  Can be a date, hash, or some other type of id.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>class</code> <code>`openml.OpenMLDataset`</code> <code>Dataset description.</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def create_dataset(  # noqa: C901, PLR0912, PLR0915\n    name: str,\n    description: str | None,\n    creator: str | None,\n    contributor: str | None,\n    collection_date: str | None,\n    language: str | None,\n    licence: str | None,\n    # TODO(eddiebergman): Docstring says `type` but I don't know what this is other than strings\n    # Edit: Found it could also be like [\"True\", \"False\"]\n    attributes: list[tuple[str, str | list[str]]] | dict[str, str | list[str]] | Literal[\"auto\"],\n    data: pd.DataFrame | np.ndarray | scipy.sparse.coo_matrix,\n    # TODO(eddiebergman): Function requires `default_target_attribute` exist but API allows None\n    default_target_attribute: str,\n    ignore_attribute: str | list[str] | None,\n    citation: str,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n    update_comment: str | None = None,\n    version_label: str | None = None,\n) -&gt; OpenMLDataset:\n    \"\"\"Create a dataset.\n\n    This function creates an OpenMLDataset object.\n    The OpenMLDataset object contains information related to the dataset\n    and the actual data file.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str\n        License of the data.\n    attributes : list, dict, or 'auto'\n        A list of tuples. Each tuple consists of the attribute name and type.\n        If passing a pandas DataFrame, the attributes can be automatically\n        inferred by passing ``'auto'``. Specific attributes can be manually\n        specified by a passing a dictionary where the key is the name of the\n        attribute and the value is the data type of the attribute.\n    data : ndarray, list, dataframe, coo_matrix, shape (n_samples, n_features)\n        An array that contains both the attributes and the targets. When\n        providing a dataframe, the attribute names and type can be inferred by\n        passing ``attributes='auto'``.\n        The target feature is indicated as meta-data of the dataset.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n        Can have multiple values, comma separated.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    version_label : str, optional\n        Version label provided by user.\n         Can be a date, hash, or some other type of id.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n\n    Returns\n    -------\n    class:`openml.OpenMLDataset`\n    Dataset description.\n    \"\"\"\n    if isinstance(data, pd.DataFrame):\n        # infer the row id from the index of the dataset\n        if row_id_attribute is None:\n            row_id_attribute = data.index.name\n        # When calling data.values, the index will be skipped.\n        # We need to reset the index such that it is part of the data.\n        if data.index.name is not None:\n            data = data.reset_index()\n\n    if attributes == \"auto\" or isinstance(attributes, dict):\n        if not isinstance(data, pd.DataFrame):\n            raise ValueError(\n                \"Automatically inferring attributes requires \"\n                f\"a pandas DataFrame. A {data!r} was given instead.\",\n            )\n        # infer the type of data for each column of the DataFrame\n        attributes_ = attributes_arff_from_df(data)\n        if isinstance(attributes, dict):\n            # override the attributes which was specified by the user\n            for attr_idx in range(len(attributes_)):\n                attr_name = attributes_[attr_idx][0]\n                if attr_name in attributes:\n                    attributes_[attr_idx] = (attr_name, attributes[attr_name])\n    else:\n        attributes_ = attributes\n    ignore_attributes = _expand_parameter(ignore_attribute)\n    _validated_data_attributes(ignore_attributes, attributes_, \"ignore_attribute\")\n\n    default_target_attributes = _expand_parameter(default_target_attribute)\n    _validated_data_attributes(default_target_attributes, attributes_, \"default_target_attribute\")\n\n    if row_id_attribute is not None:\n        is_row_id_an_attribute = any(attr[0] == row_id_attribute for attr in attributes_)\n        if not is_row_id_an_attribute:\n            raise ValueError(\n                \"'row_id_attribute' should be one of the data attribute. \"\n                \" Got '{}' while candidates are {}.\".format(\n                    row_id_attribute,\n                    [attr[0] for attr in attributes_],\n                ),\n            )\n\n    if isinstance(data, pd.DataFrame):\n        if all(isinstance(dtype, pd.SparseDtype) for dtype in data.dtypes):\n            data = data.sparse.to_coo()\n            # liac-arff only support COO matrices with sorted rows\n            row_idx_sorted = np.argsort(data.row)  # type: ignore\n            data.row = data.row[row_idx_sorted]  # type: ignore\n            data.col = data.col[row_idx_sorted]  # type: ignore\n            data.data = data.data[row_idx_sorted]  # type: ignore\n        else:\n            data = data.to_numpy()\n\n    data_format: Literal[\"arff\", \"sparse_arff\"]\n    if isinstance(data, (list, np.ndarray)):\n        if isinstance(data[0], (list, np.ndarray)):\n            data_format = \"arff\"\n        elif isinstance(data[0], dict):\n            data_format = \"sparse_arff\"\n        else:\n            raise ValueError(\n                \"When giving a list or a numpy.ndarray, \"\n                \"they should contain a list/ numpy.ndarray \"\n                \"for dense data or a dictionary for sparse \"\n                f\"data. Got {data[0]!r} instead.\",\n            )\n    elif isinstance(data, coo_matrix):\n        data_format = \"sparse_arff\"\n    else:\n        raise ValueError(\n            \"When giving a list or a numpy.ndarray, \"\n            \"they should contain a list/ numpy.ndarray \"\n            \"for dense data or a dictionary for sparse \"\n            f\"data. Got {data[0]!r} instead.\",\n        )\n\n    arff_object = {\n        \"relation\": name,\n        \"description\": description,\n        \"attributes\": attributes_,\n        \"data\": data,\n    }\n\n    # serializes the ARFF dataset object and returns a string\n    arff_dataset = arff.dumps(arff_object)\n    try:\n        # check if ARFF is valid\n        decoder = arff.ArffDecoder()\n        return_type = arff.COO if data_format == \"sparse_arff\" else arff.DENSE\n        decoder.decode(arff_dataset, encode_nominal=True, return_type=return_type)\n    except arff.ArffException as e:\n        raise ValueError(\n            \"The arguments you have provided do not construct a valid ARFF file\"\n        ) from e\n\n    return OpenMLDataset(\n        name=name,\n        description=description,\n        data_format=data_format,\n        creator=creator,\n        contributor=contributor,\n        collection_date=collection_date,\n        language=language,\n        licence=licence,\n        default_target_attribute=default_target_attribute,\n        row_id_attribute=row_id_attribute,\n        ignore_attribute=ignore_attribute,\n        citation=citation,\n        version_label=version_label,\n        original_data_url=original_data_url,\n        paper_url=paper_url,\n        update_comment=update_comment,\n        dataset=arff_dataset,\n    )\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.data_feature_add_ontology","title":"<code>data_feature_add_ontology(data_id, index, ontology)</code>","text":"<p>An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided. Adds an ontology (URL) to a given dataset feature (defined by a dataset id and index). The dataset has to exists on OpenML and needs to have been processed by the evaluation engine.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to which the feature belongs</p> required <code>index</code> <code>int</code> <p>index of the feature in dataset (0-based)</p> required <code>ontology</code> <code>str</code> <p>URL to ontology (max. 256 characters)</p> required <p>Returns:</p> Type Description <code>True or throws an OpenML server exception</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def data_feature_add_ontology(data_id: int, index: int, ontology: str) -&gt; bool:\n    \"\"\"\n    An ontology describes the concept that are described in a feature. An\n    ontology is defined by an URL where the information is provided. Adds\n    an ontology (URL) to a given dataset feature (defined by a dataset id\n    and index). The dataset has to exists on OpenML and needs to have been\n    processed by the evaluation engine.\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to which the feature belongs\n    index : int\n        index of the feature in dataset (0-based)\n    ontology : str\n        URL to ontology (max. 256 characters)\n\n    Returns\n    -------\n    True or throws an OpenML server exception\n    \"\"\"\n    upload_data: dict[str, int | str] = {\"data_id\": data_id, \"index\": index, \"ontology\": ontology}\n    openml._api_calls._perform_api_call(\"data/feature/ontology/add\", \"post\", data=upload_data)\n    # an error will be thrown in case the request was unsuccessful\n    return True\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.data_feature_remove_ontology","title":"<code>data_feature_remove_ontology(data_id, index, ontology)</code>","text":"<p>Removes an existing ontology (URL) from a given dataset feature (defined by a dataset id and index). The dataset has to exists on OpenML and needs to have been processed by the evaluation engine. Ontology needs to be attached to the specific fearure.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to which the feature belongs</p> required <code>index</code> <code>int</code> <p>index of the feature in dataset (0-based)</p> required <code>ontology</code> <code>str</code> <p>URL to ontology (max. 256 characters)</p> required <p>Returns:</p> Type Description <code>True or throws an OpenML server exception</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def data_feature_remove_ontology(data_id: int, index: int, ontology: str) -&gt; bool:\n    \"\"\"\n    Removes an existing ontology (URL) from a given dataset feature (defined\n    by a dataset id and index). The dataset has to exists on OpenML and needs\n    to have been processed by the evaluation engine. Ontology needs to be\n    attached to the specific fearure.\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to which the feature belongs\n    index : int\n        index of the feature in dataset (0-based)\n    ontology : str\n        URL to ontology (max. 256 characters)\n\n    Returns\n    -------\n    True or throws an OpenML server exception\n    \"\"\"\n    upload_data: dict[str, int | str] = {\"data_id\": data_id, \"index\": index, \"ontology\": ontology}\n    openml._api_calls._perform_api_call(\"data/feature/ontology/remove\", \"post\", data=upload_data)\n    # an error will be thrown in case the request was unsuccessful\n    return True\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.delete_dataset","title":"<code>delete_dataset(dataset_id)</code>","text":"<p>Delete dataset with id <code>dataset_id</code> from the OpenML server.</p> <p>This can only be done if you are the owner of the dataset and no tasks are attached to the dataset.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int</code> <p>OpenML id of the dataset</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def delete_dataset(dataset_id: int) -&gt; bool:\n    \"\"\"Delete dataset with id `dataset_id` from the OpenML server.\n\n    This can only be done if you are the owner of the dataset and\n    no tasks are attached to the dataset.\n\n    Parameters\n    ----------\n    dataset_id : int\n        OpenML id of the dataset\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"data\", dataset_id)\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.edit_dataset","title":"<code>edit_dataset(data_id, description=None, creator=None, contributor=None, collection_date=None, language=None, default_target_attribute=None, ignore_attribute=None, citation=None, row_id_attribute=None, original_data_url=None, paper_url=None)</code>","text":"<p>Edits an OpenMLDataset.</p> <p>In addition to providing the dataset id of the dataset to edit (through data_id), you must specify a value for at least one of the optional function arguments, i.e. one value for a field to edit.</p> <p>This function allows editing of both non-critical and critical fields. Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.</p> <ul> <li>Editing non-critical data fields is allowed for all authenticated users.</li> <li>Editing critical fields is allowed only for the owner, provided there are no tasks    associated with this dataset.</li> </ul> <p>If dataset has tasks or if the user is not the owner, the only way to edit critical fields is to use fork_dataset followed by edit_dataset.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>ID of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dataset id</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def edit_dataset(\n    data_id: int,\n    description: str | None = None,\n    creator: str | None = None,\n    contributor: str | None = None,\n    collection_date: str | None = None,\n    language: str | None = None,\n    default_target_attribute: str | None = None,\n    ignore_attribute: str | list[str] | None = None,\n    citation: str | None = None,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n) -&gt; int:\n    \"\"\"Edits an OpenMLDataset.\n\n    In addition to providing the dataset id of the dataset to edit (through data_id),\n    you must specify a value for at least one of the optional function arguments,\n    i.e. one value for a field to edit.\n\n    This function allows editing of both non-critical and critical fields.\n    Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.\n\n     - Editing non-critical data fields is allowed for all authenticated users.\n     - Editing critical fields is allowed only for the owner, provided there are no tasks\n       associated with this dataset.\n\n    If dataset has tasks or if the user is not the owner, the only way\n    to edit critical fields is to use fork_dataset followed by edit_dataset.\n\n    Parameters\n    ----------\n    data_id : int\n        ID of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n\n    Returns\n    -------\n    Dataset id\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n\n    # compose data edit parameters as xml\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    xml = OrderedDict()  # type: 'OrderedDict[str, OrderedDict]'\n    xml[\"oml:data_edit_parameters\"] = OrderedDict()\n    xml[\"oml:data_edit_parameters\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n    xml[\"oml:data_edit_parameters\"][\"oml:description\"] = description\n    xml[\"oml:data_edit_parameters\"][\"oml:creator\"] = creator\n    xml[\"oml:data_edit_parameters\"][\"oml:contributor\"] = contributor\n    xml[\"oml:data_edit_parameters\"][\"oml:collection_date\"] = collection_date\n    xml[\"oml:data_edit_parameters\"][\"oml:language\"] = language\n    xml[\"oml:data_edit_parameters\"][\"oml:default_target_attribute\"] = default_target_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:row_id_attribute\"] = row_id_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:ignore_attribute\"] = ignore_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:citation\"] = citation\n    xml[\"oml:data_edit_parameters\"][\"oml:original_data_url\"] = original_data_url\n    xml[\"oml:data_edit_parameters\"][\"oml:paper_url\"] = paper_url\n\n    # delete None inputs\n    for k in list(xml[\"oml:data_edit_parameters\"]):\n        if not xml[\"oml:data_edit_parameters\"][k]:\n            del xml[\"oml:data_edit_parameters\"][k]\n\n    file_elements = {\n        \"edit_parameters\": (\"description.xml\", xmltodict.unparse(xml)),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        \"data/edit\",\n        \"post\",\n        data=form_data,\n        file_elements=file_elements,\n    )\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_edit\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.fork_dataset","title":"<code>fork_dataset(data_id)</code>","text":"<p>Creates a new dataset version, with the authenticated user as the new owner.  The forked dataset can have distinct dataset meta-data,  but the actual data itself is shared with the original version.</p> <p>This API is intended for use when a user is unable to edit the critical fields of a dataset  through the edit_dataset API.  (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)</p> <p>Specifically, this happens when the user is:         1. Not the owner of the dataset.         2. User is the owner of the dataset, but the dataset has tasks.</p> <p>In these two cases the only way to edit critical fields is:         1. STEP 1: Fork the dataset using fork_dataset API         2. STEP 2: Call edit_dataset API on the forked version.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to be forked</p> required <p>Returns:</p> Type Description <code>Dataset id of the forked dataset</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def fork_dataset(data_id: int) -&gt; int:\n    \"\"\"\n     Creates a new dataset version, with the authenticated user as the new owner.\n     The forked dataset can have distinct dataset meta-data,\n     but the actual data itself is shared with the original version.\n\n     This API is intended for use when a user is unable to edit the critical fields of a dataset\n     through the edit_dataset API.\n     (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)\n\n     Specifically, this happens when the user is:\n            1. Not the owner of the dataset.\n            2. User is the owner of the dataset, but the dataset has tasks.\n\n     In these two cases the only way to edit critical fields is:\n            1. STEP 1: Fork the dataset using fork_dataset API\n            2. STEP 2: Call edit_dataset API on the forked version.\n\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to be forked\n\n    Returns\n    -------\n    Dataset id of the forked dataset\n\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n    # compose data fork parameters\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"data/fork\", \"post\", data=form_data)\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_fork\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.get_dataset","title":"<code>get_dataset(dataset_id, download_data=None, version=None, error_if_multiple=False, cache_format='pickle', download_qualities=None, download_features_meta_data=None, download_all_files=False, force_refresh_cache=False)</code>","text":"<p>Download the OpenML dataset representation, optionally also download actual data file.</p> <p>This function is by default NOT thread/multiprocessing safe, as this function uses caching. A check will be performed to determine if the information has previously been downloaded to a cache, and if so be loaded from disk instead of retrieved from the server.</p> <p>To make this function thread safe, you can install the python package <code>oslo.concurrency</code>. If <code>oslo.concurrency</code> is installed <code>get_dataset</code> becomes thread safe.</p> <p>Alternatively, to make this function thread/multiprocessing safe initialize the cache first by calling <code>get_dataset(args)</code> once before calling <code>get_dataset(args)</code> many times in parallel. This will initialize the cache and later calls will use the cache in a thread/multiprocessing safe way.</p> <p>If dataset is retrieved by name, a version may be specified. If no version is specified and multiple versions of the dataset exist, the earliest version of the dataset that is still active will be returned. If no version is specified, multiple versions of the dataset exist and <code>exception_if_multiple</code> is set to <code>True</code>, this function will raise an exception.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int or str</code> <p>Dataset ID of the dataset to download</p> required <code>download_data</code> <code>bool(default=True)</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>None</code> <code>version</code> <code>(int, optional(default=None))</code> <p>Specifies the version if <code>dataset_id</code> is specified by name. If no version is specified, retrieve the least recent still active version.</p> <code>None</code> <code>error_if_multiple</code> <code>bool(default=False)</code> <p>If <code>True</code> raise an error if multiple datasets are found with matching criteria.</p> <code>False</code> <code>cache_format</code> <code>str(default='pickle') in {'pickle', 'feather'}</code> <p>Format for caching the dataset - may be feather or pickle Note that the default 'pickle' option may load slower than feather when no.of.rows is very high.</p> <code>'pickle'</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description. If True, download and cache the qualities file. If False, create the OpenMLDataset without qualities metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(qualities=True)</code> method.</p> <code>None</code> <code>download_features_meta_data</code> <code>bool(default=True)</code> <p>Option to download 'features' meta-data in addition to the minimal dataset description. If True, download and cache the features file. If False, create the OpenMLDataset without features metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(features=True)</code> method.</p> <code>None</code> <code>download_all_files</code> <code>bool</code> <p>EXPERIMENTAL. Download all files related to the dataset that reside on the server. Useful for datasets which refer to auxiliary files (e.g., meta-album).</p> <code>False</code> <code>force_refresh_cache</code> <code>bool(default=False)</code> <p>Force the cache to refreshed by deleting the cache directory and re-downloading the data. Note, if <code>force_refresh_cache</code> is True, <code>get_dataset</code> is NOT thread/multiprocessing safe, because this creates a race condition to creating and deleting the cache; as in general with the cache.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>dataset</code> <code>:class:`openml.OpenMLDataset`</code> <p>The downloaded dataset.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_dataset(  # noqa: C901, PLR0912\n    dataset_id: int | str,\n    download_data: bool | None = None,  # Optional for deprecation warning; later again only bool\n    version: int | None = None,\n    error_if_multiple: bool = False,  # noqa: FBT002, FBT001\n    cache_format: Literal[\"pickle\", \"feather\"] = \"pickle\",\n    download_qualities: bool | None = None,  # Same as above\n    download_features_meta_data: bool | None = None,  # Same as above\n    download_all_files: bool = False,  # noqa: FBT002, FBT001\n    force_refresh_cache: bool = False,  # noqa: FBT001, FBT002\n) -&gt; OpenMLDataset:\n    \"\"\"Download the OpenML dataset representation, optionally also download actual data file.\n\n    This function is by default NOT thread/multiprocessing safe, as this function uses caching.\n    A check will be performed to determine if the information has previously been downloaded to a\n    cache, and if so be loaded from disk instead of retrieved from the server.\n\n    To make this function thread safe, you can install the python package ``oslo.concurrency``.\n    If ``oslo.concurrency`` is installed `get_dataset` becomes thread safe.\n\n    Alternatively, to make this function thread/multiprocessing safe initialize the cache first by\n    calling `get_dataset(args)` once before calling `get_dataset(args)` many times in parallel.\n    This will initialize the cache and later calls will use the cache in a thread/multiprocessing\n    safe way.\n\n    If dataset is retrieved by name, a version may be specified.\n    If no version is specified and multiple versions of the dataset exist,\n    the earliest version of the dataset that is still active will be returned.\n    If no version is specified, multiple versions of the dataset exist and\n    ``exception_if_multiple`` is set to ``True``, this function will raise an exception.\n\n    Parameters\n    ----------\n    dataset_id : int or str\n        Dataset ID of the dataset to download\n    download_data : bool (default=True)\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    version : int, optional (default=None)\n        Specifies the version if `dataset_id` is specified by name.\n        If no version is specified, retrieve the least recent still active version.\n    error_if_multiple : bool (default=False)\n        If ``True`` raise an error if multiple datasets are found with matching criteria.\n    cache_format : str (default='pickle') in {'pickle', 'feather'}\n        Format for caching the dataset - may be feather or pickle\n        Note that the default 'pickle' option may load slower than feather when\n        no.of.rows is very high.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n        If True, download and cache the qualities file.\n        If False, create the OpenMLDataset without qualities metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(qualities=True)` method.\n    download_features_meta_data : bool (default=True)\n        Option to download 'features' meta-data in addition to the minimal dataset description.\n        If True, download and cache the features file.\n        If False, create the OpenMLDataset without features metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(features=True)` method.\n    download_all_files: bool (default=False)\n        EXPERIMENTAL. Download all files related to the dataset that reside on the server.\n        Useful for datasets which refer to auxiliary files (e.g., meta-album).\n    force_refresh_cache : bool (default=False)\n        Force the cache to refreshed by deleting the cache directory and re-downloading the data.\n        Note, if `force_refresh_cache` is True, `get_dataset` is NOT thread/multiprocessing safe,\n        because this creates a race condition to creating and deleting the cache; as in general with\n        the cache.\n\n    Returns\n    -------\n    dataset : :class:`openml.OpenMLDataset`\n        The downloaded dataset.\n    \"\"\"\n    # TODO(0.15): Remove the deprecation warning and make the default False; adjust types above\n    #   and documentation. Also remove None-to-True-cases below\n    if any(\n        download_flag is None\n        for download_flag in [download_data, download_qualities, download_features_meta_data]\n    ):\n        warnings.warn(\n            \"Starting from Version 0.15 `download_data`, `download_qualities`, and `download_featu\"\n            \"res_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy \"\n            \"loading. To disable this message until version 0.15 explicitly set `download_data`, \"\n            \"`download_qualities`, and `download_features_meta_data` to a bool while calling \"\n            \"`get_dataset`.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    download_data = True if download_data is None else download_data\n    download_qualities = True if download_qualities is None else download_qualities\n    download_features_meta_data = (\n        True if download_features_meta_data is None else download_features_meta_data\n    )\n\n    if download_all_files:\n        warnings.warn(\n            \"``download_all_files`` is experimental and is likely to break with new releases.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    if cache_format not in [\"feather\", \"pickle\"]:\n        raise ValueError(\n            \"cache_format must be one of 'feather' or 'pickle. \"\n            f\"Invalid format specified: {cache_format}\",\n        )\n\n    if isinstance(dataset_id, str):\n        try:\n            dataset_id = int(dataset_id)\n        except ValueError:\n            dataset_id = _name_to_id(dataset_id, version, error_if_multiple)  # type: ignore\n    elif not isinstance(dataset_id, int):\n        raise TypeError(\n            f\"`dataset_id` must be one of `str` or `int`, not {type(dataset_id)}.\",\n        )\n\n    if force_refresh_cache:\n        did_cache_dir = _get_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, dataset_id)\n        if did_cache_dir.exists():\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    did_cache_dir = _create_cache_directory_for_id(\n        DATASETS_CACHE_DIR_NAME,\n        dataset_id,\n    )\n\n    remove_dataset_cache = True\n    try:\n        description = _get_dataset_description(did_cache_dir, dataset_id)\n        features_file = None\n        qualities_file = None\n\n        if download_features_meta_data:\n            features_file = _get_dataset_features_file(did_cache_dir, dataset_id)\n        if download_qualities:\n            qualities_file = _get_dataset_qualities_file(did_cache_dir, dataset_id)\n\n        arff_file = _get_dataset_arff(description) if download_data else None\n        if \"oml:parquet_url\" in description and download_data:\n            try:\n                parquet_file = _get_dataset_parquet(\n                    description,\n                    download_all_files=download_all_files,\n                )\n            except urllib3.exceptions.MaxRetryError:\n                parquet_file = None\n            if parquet_file is None and arff_file:\n                logger.warning(\"Failed to download parquet, fallback on ARFF.\")\n        else:\n            parquet_file = None\n        remove_dataset_cache = False\n    except OpenMLServerException as e:\n        # if there was an exception\n        # check if the user had access to the dataset\n        if e.code == NO_ACCESS_GRANTED_ERRCODE:\n            raise OpenMLPrivateDatasetError(e.message) from None\n\n        raise e\n    finally:\n        if remove_dataset_cache:\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    return _create_dataset_from_description(\n        description,\n        features_file,\n        qualities_file,\n        arff_file,\n        parquet_file,\n        cache_format,\n    )\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.get_datasets","title":"<code>get_datasets(dataset_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download datasets.</p> <p>This function iterates :meth:<code>openml.datasets.get_dataset</code>.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>iterable</code> <p>Integers or strings representing dataset ids or dataset names. If dataset names are specified, the least recent still active dataset version is returned.</p> required <code>download_data</code> <code>bool</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>True</code> <code>download_qualities</code> <code>(bool, optional(default=True))</code> <p>If True, also download qualities.xml file. If False it skip the qualities.xml.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>list of datasets</code> <p>A list of dataset objects.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def get_datasets(\n    dataset_ids: list[str | int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLDataset]:\n    \"\"\"Download datasets.\n\n    This function iterates :meth:`openml.datasets.get_dataset`.\n\n    Parameters\n    ----------\n    dataset_ids : iterable\n        Integers or strings representing dataset ids or dataset names.\n        If dataset names are specified, the least recent still active dataset version is returned.\n    download_data : bool, optional\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    download_qualities : bool, optional (default=True)\n        If True, also download qualities.xml file. If False it skip the qualities.xml.\n\n    Returns\n    -------\n    datasets : list of datasets\n        A list of dataset objects.\n    \"\"\"\n    datasets = []\n    for dataset_id in dataset_ids:\n        datasets.append(\n            get_dataset(dataset_id, download_data, download_qualities=download_qualities),\n        )\n    return datasets\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.list_datasets","title":"<code>list_datasets(data_id=None, offset=None, size=None, status=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None, offset: int | None, size: int | None, status: str | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all dataset which are on OpenML. Supports large amount of results.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>list</code> <p>A list of data ids, to specify which datasets should be listed</p> <code>None</code> <code>offset</code> <code>int</code> <p>The number of datasets to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of datasets to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated}. By default active datasets are returned, but also datasets from another status can be requested.</p> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>output_format</code> <code>Literal['dataframe', 'dict']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators (keys in the dict): data_name, data_version, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from dataset ID to dict.</p> <p>Every dataset is represented by a dictionary containing the following information: - dataset id - name - format - status If qualities are calculated for the dataset, some of these are also returned.</p> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>dataset id</li> <li>name</li> <li>format</li> <li>status If qualities are calculated for the dataset, some of these are also included as columns.</li> </ul> </li> </ul> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_datasets(\n    data_id: list[int] | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dataframe\", \"dict\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all dataset which are on OpenML.\n    Supports large amount of results.\n\n    Parameters\n    ----------\n    data_id : list, optional\n        A list of data ids, to specify which datasets should be\n        listed\n    offset : int, optional\n        The number of datasets to skip, starting from the first.\n    size : int, optional\n        The maximum number of datasets to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated}. By\n        default active datasets are returned, but also datasets\n        from another status can be requested.\n    tag : str, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs : dict, optional\n        Legal filter operators (keys in the dict):\n        data_name, data_version, number_instances,\n        number_features, number_classes, number_missing_values.\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from dataset ID to dict.\n\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also included as columns.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        data_id=data_id,\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_datasets,\n        offset=offset,\n        size=size,\n        status=status,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.list_qualities","title":"<code>list_qualities()</code>","text":"<p>Return list of data qualities available.</p> <p>The function performs an API call to retrieve the entire list of data qualities that are computed on the datasets uploaded.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_qualities() -&gt; list[str]:\n    \"\"\"Return list of data qualities available.\n\n    The function performs an API call to retrieve the entire list of\n    data qualities that are computed on the datasets uploaded.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    api_call = \"data/qualities/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:quality\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:data_qualities_list\" not in qualities:\n        raise ValueError('Error in return XML, does not contain \"oml:data_qualities_list\"')\n\n    if not isinstance(qualities[\"oml:data_qualities_list\"][\"oml:quality\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:quality\" as a list')\n\n    return qualities[\"oml:data_qualities_list\"][\"oml:quality\"]\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.status_update","title":"<code>status_update(data_id, status)</code>","text":"<p>Updates the status of a dataset to either 'active' or 'deactivated'. Please see the OpenML API documentation for a description of the status and all legal status transitions: https://docs.openml.org/#dataset-status</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/datasets/functions.py</code> <pre><code>def status_update(data_id: int, status: Literal[\"active\", \"deactivated\"]) -&gt; None:\n    \"\"\"\n    Updates the status of a dataset to either 'active' or 'deactivated'.\n    Please see the OpenML API documentation for a description of the status\n    and all legal status transitions:\n    https://docs.openml.org/#dataset-status\n\n    Parameters\n    ----------\n    data_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(f\"Illegal status value. Legal values: {legal_status}\")\n\n    data: openml._api_calls.DATA_TYPE = {\"data_id\": data_id, \"status\": status}\n    result_xml = openml._api_calls._perform_api_call(\"data/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_data_id = result[\"oml:data_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:data_status_update\"][\"oml:status\"]\n    if status != server_status or int(data_id) != int(server_data_id):\n        # This should never happen\n        raise ValueError(\"Data id/status does not collide\")\n</code></pre>"},{"location":"reference/evaluations/","title":"evaluations","text":""},{"location":"reference/evaluations/#openml.evaluations.OpenMLEvaluation","title":"<code>OpenMLEvaluation</code>","text":"<p>Contains all meta-information about a run / evaluation combination, according to the evaluation/list function</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Refers to the run.</p> required <code>task_id</code> <code>int</code> <p>Refers to the task.</p> required <code>setup_id</code> <code>int</code> <p>Refers to the setup.</p> required <code>flow_id</code> <code>int</code> <p>Refers to the flow.</p> required <code>flow_name</code> <code>str</code> <p>Name of the referred flow.</p> required <code>data_id</code> <code>int</code> <p>Refers to the dataset.</p> required <code>data_name</code> <code>str</code> <p>The name of the dataset.</p> required <code>function</code> <code>str</code> <p>The evaluation metric of this item (e.g., accuracy).</p> required <code>upload_time</code> <code>str</code> <p>The time of evaluation.</p> required <code>uploader</code> <code>int</code> <p>Uploader ID (user ID)</p> required <code>upload_name</code> <code>str</code> <p>Name of the uploader of this evaluation</p> required <code>value</code> <code>float</code> <p>The value (score) of this evaluation.</p> required <code>values</code> <code>List[float]</code> <p>The values (scores) per repeat and fold (if requested)</p> required <code>array_data</code> <code>str</code> <p>list of information per class. (e.g., in case of precision, auroc, recall)</p> <code>None</code> Source code in <code>openml/evaluations/evaluation.py</code> <pre><code>class OpenMLEvaluation:\n    \"\"\"\n    Contains all meta-information about a run / evaluation combination,\n    according to the evaluation/list function\n\n    Parameters\n    ----------\n    run_id : int\n        Refers to the run.\n    task_id : int\n        Refers to the task.\n    setup_id : int\n        Refers to the setup.\n    flow_id : int\n        Refers to the flow.\n    flow_name : str\n        Name of the referred flow.\n    data_id : int\n        Refers to the dataset.\n    data_name : str\n        The name of the dataset.\n    function : str\n        The evaluation metric of this item (e.g., accuracy).\n    upload_time : str\n        The time of evaluation.\n    uploader: int\n        Uploader ID (user ID)\n    upload_name : str\n        Name of the uploader of this evaluation\n    value : float\n        The value (score) of this evaluation.\n    values : List[float]\n        The values (scores) per repeat and fold (if requested)\n    array_data : str\n        list of information per class.\n        (e.g., in case of precision, auroc, recall)\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        run_id: int,\n        task_id: int,\n        setup_id: int,\n        flow_id: int,\n        flow_name: str,\n        data_id: int,\n        data_name: str,\n        function: str,\n        upload_time: str,\n        uploader: int,\n        uploader_name: str,\n        value: float | None,\n        values: list[float] | None,\n        array_data: str | None = None,\n    ):\n        self.run_id = run_id\n        self.task_id = task_id\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.data_id = data_id\n        self.data_name = data_name\n        self.function = function\n        self.upload_time = upload_time\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.value = value\n        self.values = values\n        self.array_data = array_data\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Evaluation\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Upload Date\": self.upload_time,\n            \"Run ID\": self.run_id,\n            \"OpenML Run URL\": openml.runs.OpenMLRun.url_for_id(self.run_id),\n            \"Task ID\": self.task_id,\n            \"OpenML Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"OpenML Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Setup ID\": self.setup_id,\n            \"Data ID\": self.data_id,\n            \"Data Name\": self.data_name,\n            \"OpenML Data URL\": openml.datasets.OpenMLDataset.url_for_id(self.data_id),\n            \"Metric Used\": self.function,\n            \"Result\": self.value,\n        }\n\n        order = [\n            \"Uploader Date\",\n            \"Run ID\",\n            \"OpenML Run URL\",\n            \"Task ID\",\n            \"OpenML Task URL\" \"Flow ID\",\n            \"OpenML Flow URL\",\n            \"Setup ID\",\n            \"Data ID\",\n            \"Data Name\",\n            \"OpenML Data URL\",\n            \"Metric Used\",\n            \"Result\",\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/evaluations/#openml.evaluations.list_evaluation_measures","title":"<code>list_evaluation_measures()</code>","text":"<p>Return list of evaluation measures available.</p> <p>The function performs an API call to retrieve the entire list of evaluation measures that are available.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluation_measures() -&gt; list[str]:\n    \"\"\"Return list of evaluation measures available.\n\n    The function performs an API call to retrieve the entire list of\n    evaluation measures that are available.\n\n    Returns\n    -------\n    list\n\n    \"\"\"\n    api_call = \"evaluationmeasure/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:measures\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:evaluation_measures\" not in qualities:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:evaluation_measures\"')\n    if not isinstance(qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:measure\" as a list')\n    return qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"]\n</code></pre>"},{"location":"reference/evaluations/#openml.evaluations.list_evaluations","title":"<code>list_evaluations(function, offset=None, size=10000, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, study=None, per_fold=None, sort_order=None, output_format='object')</code>","text":"<pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dict', 'object'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dataframe'] = ...) -&gt; pd.DataFrame\n</code></pre> <p>List all run-evaluation pairs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of runs to show. If set to <code>None</code>, it returns all the results.</p> <code>10000</code> <code>tasks</code> <code>list[int, str]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list[str | int] | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int, str]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[str | int] | None</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int, str]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'object' the output is a dict of OpenMLEvaluation objects - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations(\n    function: str,\n    offset: int | None = None,\n    size: int | None = 10000,\n    tasks: list[str | int] | None = None,\n    setups: list[str | int] | None = None,\n    flows: list[str | int] | None = None,\n    runs: list[str | int] | None = None,\n    uploaders: list[str | int] | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, default 10000\n        The maximum number of runs to show.\n        If set to ``None``, it returns all the results.\n\n    tasks : list[int,str], optional\n        the list of task IDs\n    setups: list[int,str], optional\n        the list of setup IDs\n    flows : list[int,str], optional\n        the list of flow IDs\n    runs :list[int,str], optional\n        the list of run IDs\n    uploaders : list[int,str], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n\n    study : int, optional\n\n    per_fold : bool, optional\n\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'object' the output is a dict of OpenMLEvaluation objects\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. Only 'object', 'dataframe', or 'dict' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    per_fold_str = None\n    if per_fold is not None:\n        per_fold_str = str(per_fold).lower()\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_evaluations,\n        function=function,\n        offset=offset,\n        size=size,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        runs=runs,\n        uploaders=uploaders,\n        tag=tag,\n        study=study,\n        sort_order=sort_order,\n        per_fold=per_fold_str,\n    )\n</code></pre>"},{"location":"reference/evaluations/#openml.evaluations.list_evaluations_setups","title":"<code>list_evaluations_setups(function, offset=None, size=None, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, per_fold=None, sort_order=None, output_format='dataframe', parameters_in_separate_columns=False)</code>","text":"<p>List all run-evaluation pairs matching all of the given filters and their hyperparameter settings.</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>tasks</code> <code>list[int]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[int]</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>str</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dataframe'</code> <code>parameters_in_separate_columns</code> <code>bool</code> <p>Returns hyperparameters in separate columns if set to True. Valid only for a single flow</p> <code>False</code> <p>Returns:</p> Type Description <code>dict or dataframe with hyperparameter settings as a list of tuples.</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations_setups(\n    function: str,\n    offset: int | None = None,\n    size: int | None = None,\n    tasks: list | None = None,\n    setups: list | None = None,\n    flows: list | None = None,\n    runs: list | None = None,\n    uploaders: list | None = None,\n    tag: str | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: str = \"dataframe\",\n    parameters_in_separate_columns: bool = False,  # noqa: FBT001, FBT002\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters\n    and their hyperparameter settings.\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n    tasks : list[int], optional\n        the list of task IDs\n    setups: list[int], optional\n        the list of setup IDs\n    flows : list[int], optional\n        the list of flow IDs\n    runs : list[int], optional\n        the list of run IDs\n    uploaders : list[int], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n    per_fold : bool, optional\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n    output_format: str, optional (default='dataframe')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    parameters_in_separate_columns: bool, optional (default= False)\n        Returns hyperparameters in separate columns if set to True.\n        Valid only for a single flow\n\n\n    Returns\n    -------\n    dict or dataframe with hyperparameter settings as a list of tuples.\n    \"\"\"\n    if parameters_in_separate_columns and (flows is None or len(flows) != 1):\n        raise ValueError(\n            \"Can set parameters_in_separate_columns to true \" \"only for single flow_id\",\n        )\n\n    # List evaluations\n    evals = list_evaluations(\n        function=function,\n        offset=offset,\n        size=size,\n        runs=runs,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        uploaders=uploaders,\n        tag=tag,\n        per_fold=per_fold,\n        sort_order=sort_order,\n        output_format=\"dataframe\",\n    )\n    # List setups\n    # list_setups by setup id does not support large sizes (exceeds URL length limit)\n    # Hence we split the list of unique setup ids returned by list_evaluations into chunks of size N\n    _df = pd.DataFrame()\n    if len(evals) != 0:\n        N = 100  # size of section\n        length = len(evals[\"setup_id\"].unique())  # length of the array we want to split\n        # array_split - allows indices_or_sections to not equally divide the array\n        # array_split -length % N sub-arrays of size length//N + 1 and the rest of size length//N.\n        uniq = np.asarray(evals[\"setup_id\"].unique())\n        setup_chunks = np.array_split(uniq, ((length - 1) // N) + 1)\n        setup_data = pd.DataFrame()\n        for _setups in setup_chunks:\n            result = openml.setups.list_setups(setup=_setups, output_format=\"dataframe\")\n            assert isinstance(result, pd.DataFrame)\n            result = result.drop(\"flow_id\", axis=1)\n            # concat resulting setup chunks into single datframe\n            setup_data = pd.concat([setup_data, result], ignore_index=True)\n\n        parameters = []\n        # Convert parameters of setup into list of tuples of (hyperparameter, value)\n        for parameter_dict in setup_data[\"parameters\"]:\n            if parameter_dict is not None:\n                parameters.append(\n                    {param[\"full_name\"]: param[\"value\"] for param in parameter_dict.values()},\n                )\n            else:\n                parameters.append({})\n        setup_data[\"parameters\"] = parameters\n        # Merge setups with evaluations\n        _df = evals.merge(setup_data, on=\"setup_id\", how=\"left\")\n\n    if parameters_in_separate_columns:\n        _df = pd.concat(\n            [_df.drop(\"parameters\", axis=1), _df[\"parameters\"].apply(pd.Series)],\n            axis=1,\n        )\n\n    if output_format == \"dataframe\":\n        return _df\n\n    return _df.to_dict(orient=\"index\")\n</code></pre>"},{"location":"reference/evaluations/evaluation/","title":"evaluation","text":""},{"location":"reference/evaluations/evaluation/#openml.evaluations.evaluation.OpenMLEvaluation","title":"<code>OpenMLEvaluation</code>","text":"<p>Contains all meta-information about a run / evaluation combination, according to the evaluation/list function</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Refers to the run.</p> required <code>task_id</code> <code>int</code> <p>Refers to the task.</p> required <code>setup_id</code> <code>int</code> <p>Refers to the setup.</p> required <code>flow_id</code> <code>int</code> <p>Refers to the flow.</p> required <code>flow_name</code> <code>str</code> <p>Name of the referred flow.</p> required <code>data_id</code> <code>int</code> <p>Refers to the dataset.</p> required <code>data_name</code> <code>str</code> <p>The name of the dataset.</p> required <code>function</code> <code>str</code> <p>The evaluation metric of this item (e.g., accuracy).</p> required <code>upload_time</code> <code>str</code> <p>The time of evaluation.</p> required <code>uploader</code> <code>int</code> <p>Uploader ID (user ID)</p> required <code>upload_name</code> <code>str</code> <p>Name of the uploader of this evaluation</p> required <code>value</code> <code>float</code> <p>The value (score) of this evaluation.</p> required <code>values</code> <code>List[float]</code> <p>The values (scores) per repeat and fold (if requested)</p> required <code>array_data</code> <code>str</code> <p>list of information per class. (e.g., in case of precision, auroc, recall)</p> <code>None</code> Source code in <code>openml/evaluations/evaluation.py</code> <pre><code>class OpenMLEvaluation:\n    \"\"\"\n    Contains all meta-information about a run / evaluation combination,\n    according to the evaluation/list function\n\n    Parameters\n    ----------\n    run_id : int\n        Refers to the run.\n    task_id : int\n        Refers to the task.\n    setup_id : int\n        Refers to the setup.\n    flow_id : int\n        Refers to the flow.\n    flow_name : str\n        Name of the referred flow.\n    data_id : int\n        Refers to the dataset.\n    data_name : str\n        The name of the dataset.\n    function : str\n        The evaluation metric of this item (e.g., accuracy).\n    upload_time : str\n        The time of evaluation.\n    uploader: int\n        Uploader ID (user ID)\n    upload_name : str\n        Name of the uploader of this evaluation\n    value : float\n        The value (score) of this evaluation.\n    values : List[float]\n        The values (scores) per repeat and fold (if requested)\n    array_data : str\n        list of information per class.\n        (e.g., in case of precision, auroc, recall)\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        run_id: int,\n        task_id: int,\n        setup_id: int,\n        flow_id: int,\n        flow_name: str,\n        data_id: int,\n        data_name: str,\n        function: str,\n        upload_time: str,\n        uploader: int,\n        uploader_name: str,\n        value: float | None,\n        values: list[float] | None,\n        array_data: str | None = None,\n    ):\n        self.run_id = run_id\n        self.task_id = task_id\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.data_id = data_id\n        self.data_name = data_name\n        self.function = function\n        self.upload_time = upload_time\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.value = value\n        self.values = values\n        self.array_data = array_data\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Evaluation\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Upload Date\": self.upload_time,\n            \"Run ID\": self.run_id,\n            \"OpenML Run URL\": openml.runs.OpenMLRun.url_for_id(self.run_id),\n            \"Task ID\": self.task_id,\n            \"OpenML Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"OpenML Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Setup ID\": self.setup_id,\n            \"Data ID\": self.data_id,\n            \"Data Name\": self.data_name,\n            \"OpenML Data URL\": openml.datasets.OpenMLDataset.url_for_id(self.data_id),\n            \"Metric Used\": self.function,\n            \"Result\": self.value,\n        }\n\n        order = [\n            \"Uploader Date\",\n            \"Run ID\",\n            \"OpenML Run URL\",\n            \"Task ID\",\n            \"OpenML Task URL\" \"Flow ID\",\n            \"OpenML Flow URL\",\n            \"Setup ID\",\n            \"Data ID\",\n            \"Data Name\",\n            \"OpenML Data URL\",\n            \"Metric Used\",\n            \"Result\",\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/evaluations/functions/","title":"functions","text":""},{"location":"reference/evaluations/functions/#openml.evaluations.functions.__list_evaluations","title":"<code>__list_evaluations(api_call, output_format='object')</code>","text":"<p>Helper function to parse API calls which are lists of runs</p> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def __list_evaluations(\n    api_call: str,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Helper function to parse API calls which are lists of runs\"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    evals_dict = xmltodict.parse(xml_string, force_list=(\"oml:evaluation\",))\n    # Minimalistic check if the XML is useful\n    if \"oml:evaluations\" not in evals_dict:\n        raise ValueError(\n            \"Error in return XML, does not contain \" '\"oml:evaluations\": %s' % str(evals_dict),\n        )\n\n    assert isinstance(evals_dict[\"oml:evaluations\"][\"oml:evaluation\"], list), type(\n        evals_dict[\"oml:evaluations\"],\n    )\n\n    evals: dict[int, dict | OpenMLEvaluation] = {}\n    uploader_ids = list(\n        {eval_[\"oml:uploader\"] for eval_ in evals_dict[\"oml:evaluations\"][\"oml:evaluation\"]},\n    )\n    api_users = \"user/list/user_id/\" + \",\".join(uploader_ids)\n    xml_string_user = openml._api_calls._perform_api_call(api_users, \"get\")\n    users = xmltodict.parse(xml_string_user, force_list=(\"oml:user\",))\n    user_dict = {user[\"oml:id\"]: user[\"oml:username\"] for user in users[\"oml:users\"][\"oml:user\"]}\n    for eval_ in evals_dict[\"oml:evaluations\"][\"oml:evaluation\"]:\n        run_id = int(eval_[\"oml:run_id\"])\n\n        value = None\n        if \"oml:value\" in eval_:\n            value = float(eval_[\"oml:value\"])\n\n        values = None\n        if \"oml:values\" in eval_:\n            values = json.loads(eval_[\"oml:values\"])\n\n        array_data = eval_.get(\"oml:array_data\")\n\n        if output_format == \"object\":\n            evals[run_id] = OpenMLEvaluation(\n                run_id=run_id,\n                task_id=int(eval_[\"oml:task_id\"]),\n                setup_id=int(eval_[\"oml:setup_id\"]),\n                flow_id=int(eval_[\"oml:flow_id\"]),\n                flow_name=eval_[\"oml:flow_name\"],\n                data_id=int(eval_[\"oml:data_id\"]),\n                data_name=eval_[\"oml:data_name\"],\n                function=eval_[\"oml:function\"],\n                upload_time=eval_[\"oml:upload_time\"],\n                uploader=int(eval_[\"oml:uploader\"]),\n                uploader_name=user_dict[eval_[\"oml:uploader\"]],\n                value=value,\n                values=values,\n                array_data=array_data,\n            )\n        else:\n            # for output_format in ['dict', 'dataframe']\n            evals[run_id] = {\n                \"run_id\": int(eval_[\"oml:run_id\"]),\n                \"task_id\": int(eval_[\"oml:task_id\"]),\n                \"setup_id\": int(eval_[\"oml:setup_id\"]),\n                \"flow_id\": int(eval_[\"oml:flow_id\"]),\n                \"flow_name\": eval_[\"oml:flow_name\"],\n                \"data_id\": int(eval_[\"oml:data_id\"]),\n                \"data_name\": eval_[\"oml:data_name\"],\n                \"function\": eval_[\"oml:function\"],\n                \"upload_time\": eval_[\"oml:upload_time\"],\n                \"uploader\": int(eval_[\"oml:uploader\"]),\n                \"uploader_name\": user_dict[eval_[\"oml:uploader\"]],\n                \"value\": value,\n                \"values\": values,\n                \"array_data\": array_data,\n            }\n\n    if output_format == \"dataframe\":\n        rows = list(evals.values())\n        return pd.DataFrame.from_records(rows, columns=rows[0].keys())  # type: ignore\n\n    return evals\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_estimation_procedures","title":"<code>list_estimation_procedures()</code>","text":"<p>Return list of evaluation procedures available.</p> <p>The function performs an API call to retrieve the entire list of evaluation procedures' names that are available.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_estimation_procedures() -&gt; list[str]:\n    \"\"\"Return list of evaluation procedures available.\n\n    The function performs an API call to retrieve the entire list of\n    evaluation procedures' names that are available.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    api_call = \"estimationprocedure/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    api_results = xmltodict.parse(xml_string)\n\n    # Minimalistic check if the XML is useful\n    if \"oml:estimationprocedures\" not in api_results:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:estimationprocedures\"')\n    if \"oml:estimationprocedure\" not in api_results[\"oml:estimationprocedures\"]:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:estimationprocedure\"')\n\n    if not isinstance(api_results[\"oml:estimationprocedures\"][\"oml:estimationprocedure\"], list):\n        raise TypeError(\n            \"Error in return XML, does not contain \" '\"oml:estimationprocedure\" as a list',\n        )\n\n    return [\n        prod[\"oml:name\"]\n        for prod in api_results[\"oml:estimationprocedures\"][\"oml:estimationprocedure\"]\n    ]\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_evaluation_measures","title":"<code>list_evaluation_measures()</code>","text":"<p>Return list of evaluation measures available.</p> <p>The function performs an API call to retrieve the entire list of evaluation measures that are available.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluation_measures() -&gt; list[str]:\n    \"\"\"Return list of evaluation measures available.\n\n    The function performs an API call to retrieve the entire list of\n    evaluation measures that are available.\n\n    Returns\n    -------\n    list\n\n    \"\"\"\n    api_call = \"evaluationmeasure/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:measures\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:evaluation_measures\" not in qualities:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:evaluation_measures\"')\n    if not isinstance(qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:measure\" as a list')\n    return qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"]\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_evaluations","title":"<code>list_evaluations(function, offset=None, size=10000, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, study=None, per_fold=None, sort_order=None, output_format='object')</code>","text":"<pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dict', 'object'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dataframe'] = ...) -&gt; pd.DataFrame\n</code></pre> <p>List all run-evaluation pairs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of runs to show. If set to <code>None</code>, it returns all the results.</p> <code>10000</code> <code>tasks</code> <code>list[int, str]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list[str | int] | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int, str]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[str | int] | None</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int, str]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'object' the output is a dict of OpenMLEvaluation objects - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations(\n    function: str,\n    offset: int | None = None,\n    size: int | None = 10000,\n    tasks: list[str | int] | None = None,\n    setups: list[str | int] | None = None,\n    flows: list[str | int] | None = None,\n    runs: list[str | int] | None = None,\n    uploaders: list[str | int] | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, default 10000\n        The maximum number of runs to show.\n        If set to ``None``, it returns all the results.\n\n    tasks : list[int,str], optional\n        the list of task IDs\n    setups: list[int,str], optional\n        the list of setup IDs\n    flows : list[int,str], optional\n        the list of flow IDs\n    runs :list[int,str], optional\n        the list of run IDs\n    uploaders : list[int,str], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n\n    study : int, optional\n\n    per_fold : bool, optional\n\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'object' the output is a dict of OpenMLEvaluation objects\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. Only 'object', 'dataframe', or 'dict' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    per_fold_str = None\n    if per_fold is not None:\n        per_fold_str = str(per_fold).lower()\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_evaluations,\n        function=function,\n        offset=offset,\n        size=size,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        runs=runs,\n        uploaders=uploaders,\n        tag=tag,\n        study=study,\n        sort_order=sort_order,\n        per_fold=per_fold_str,\n    )\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_evaluations_setups","title":"<code>list_evaluations_setups(function, offset=None, size=None, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, per_fold=None, sort_order=None, output_format='dataframe', parameters_in_separate_columns=False)</code>","text":"<p>List all run-evaluation pairs matching all of the given filters and their hyperparameter settings.</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>tasks</code> <code>list[int]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[int]</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>str</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dataframe'</code> <code>parameters_in_separate_columns</code> <code>bool</code> <p>Returns hyperparameters in separate columns if set to True. Valid only for a single flow</p> <code>False</code> <p>Returns:</p> Type Description <code>dict or dataframe with hyperparameter settings as a list of tuples.</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations_setups(\n    function: str,\n    offset: int | None = None,\n    size: int | None = None,\n    tasks: list | None = None,\n    setups: list | None = None,\n    flows: list | None = None,\n    runs: list | None = None,\n    uploaders: list | None = None,\n    tag: str | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: str = \"dataframe\",\n    parameters_in_separate_columns: bool = False,  # noqa: FBT001, FBT002\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters\n    and their hyperparameter settings.\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n    tasks : list[int], optional\n        the list of task IDs\n    setups: list[int], optional\n        the list of setup IDs\n    flows : list[int], optional\n        the list of flow IDs\n    runs : list[int], optional\n        the list of run IDs\n    uploaders : list[int], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n    per_fold : bool, optional\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n    output_format: str, optional (default='dataframe')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    parameters_in_separate_columns: bool, optional (default= False)\n        Returns hyperparameters in separate columns if set to True.\n        Valid only for a single flow\n\n\n    Returns\n    -------\n    dict or dataframe with hyperparameter settings as a list of tuples.\n    \"\"\"\n    if parameters_in_separate_columns and (flows is None or len(flows) != 1):\n        raise ValueError(\n            \"Can set parameters_in_separate_columns to true \" \"only for single flow_id\",\n        )\n\n    # List evaluations\n    evals = list_evaluations(\n        function=function,\n        offset=offset,\n        size=size,\n        runs=runs,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        uploaders=uploaders,\n        tag=tag,\n        per_fold=per_fold,\n        sort_order=sort_order,\n        output_format=\"dataframe\",\n    )\n    # List setups\n    # list_setups by setup id does not support large sizes (exceeds URL length limit)\n    # Hence we split the list of unique setup ids returned by list_evaluations into chunks of size N\n    _df = pd.DataFrame()\n    if len(evals) != 0:\n        N = 100  # size of section\n        length = len(evals[\"setup_id\"].unique())  # length of the array we want to split\n        # array_split - allows indices_or_sections to not equally divide the array\n        # array_split -length % N sub-arrays of size length//N + 1 and the rest of size length//N.\n        uniq = np.asarray(evals[\"setup_id\"].unique())\n        setup_chunks = np.array_split(uniq, ((length - 1) // N) + 1)\n        setup_data = pd.DataFrame()\n        for _setups in setup_chunks:\n            result = openml.setups.list_setups(setup=_setups, output_format=\"dataframe\")\n            assert isinstance(result, pd.DataFrame)\n            result = result.drop(\"flow_id\", axis=1)\n            # concat resulting setup chunks into single datframe\n            setup_data = pd.concat([setup_data, result], ignore_index=True)\n\n        parameters = []\n        # Convert parameters of setup into list of tuples of (hyperparameter, value)\n        for parameter_dict in setup_data[\"parameters\"]:\n            if parameter_dict is not None:\n                parameters.append(\n                    {param[\"full_name\"]: param[\"value\"] for param in parameter_dict.values()},\n                )\n            else:\n                parameters.append({})\n        setup_data[\"parameters\"] = parameters\n        # Merge setups with evaluations\n        _df = evals.merge(setup_data, on=\"setup_id\", how=\"left\")\n\n    if parameters_in_separate_columns:\n        _df = pd.concat(\n            [_df.drop(\"parameters\", axis=1), _df[\"parameters\"].apply(pd.Series)],\n            axis=1,\n        )\n\n    if output_format == \"dataframe\":\n        return _df\n\n    return _df.to_dict(orient=\"index\")\n</code></pre>"},{"location":"reference/extensions/","title":"extensions","text":""},{"location":"reference/extensions/#openml.extensions.Extension","title":"<code>Extension</code>","text":"<p>               Bases: <code>ABC</code></p> <p>Defines the interface to connect machine learning libraries to OpenML-Python.</p> <p>See <code>openml.extension.sklearn.extension</code> for an implementation to bootstrap from.</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>class Extension(ABC):\n    \"\"\"Defines the interface to connect machine learning libraries to OpenML-Python.\n\n    See ``openml.extension.sklearn.extension`` for an implementation to bootstrap from.\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    @abstractmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given flow can be handled by this extension.\n\n        This is typically done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @classmethod\n    @abstractmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model flow can be handled by this extension.\n\n        This is typically done by checking the type of the model, or the package it belongs to.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for flow serialization and de-serialization\n\n    @abstractmethod\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Instantiate a model from the flow representation.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n    @abstractmethod\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n\n    @abstractmethod\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Returns\n        -------\n        List\n        \"\"\"\n\n    @abstractmethod\n    def create_setup_string(self, model: Any) -&gt; str:\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for performing runs with extension modules\n\n    @abstractmethod\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is an estimator for the given extension.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @abstractmethod\n    def seed_model(self, model: Any, seed: int | None) -&gt; Any:\n        \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Parameters\n        ----------\n        model : Any\n            The model to be seeded\n        seed : int\n\n        Returns\n        -------\n        model\n        \"\"\"\n\n    @abstractmethod\n    def _run_model_on_fold(  # noqa: PLR0913\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | None = None,\n    ) -&gt; tuple[np.ndarray, np.ndarray | None, OrderedDict[str, float], OpenMLRunTrace | None]:\n        \"\"\"Run a model on a repeat, fold, subsample triplet of the task.\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        :func:`openml.runs.run_flow_on_task`.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        predictions : np.ndarray\n            Model predictions.\n        probabilities :  Optional, np.ndarray\n            Predicted probabilities (only applicable for supervised classification tasks).\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : Optional, OpenMLRunTrace\n            Hyperparameter optimization trace (only applicable for supervised tasks with\n            hyperparameter optimization).\n        \"\"\"\n\n    @abstractmethod\n    def obtain_parameter_values(\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n\n    @abstractmethod\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for hyperparameter optimization\n\n    @abstractmethod\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n        model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a given flow can be handled by this extension.</p> <p>This is typically done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given flow can be handled by this extension.\n\n    This is typically done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a model flow can be handled by this extension.</p> <p>This is typically done by checking the type of the model, or the package it belongs to.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model flow can be handled by this extension.\n\n    This is typically done by checking the type of the model, or the package it belongs to.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>  <code>abstractmethod</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.create_setup_string","title":"<code>create_setup_string(model)</code>  <code>abstractmethod</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef create_setup_string(self, model: Any) -&gt; str:\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a model from the flow representation.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT002, FBT001\n) -&gt; Any:\n    \"\"\"Instantiate a model from the flow representation.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.get_version_information","title":"<code>get_version_information()</code>  <code>abstractmethod</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Returns\n    -------\n    List\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a base model which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n    model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.is_estimator","title":"<code>is_estimator(model)</code>  <code>abstractmethod</code>","text":"<p>Check whether the given model is an estimator for the given extension.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is an estimator for the given extension.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.model_to_flow","title":"<code>model_to_flow(model)</code>  <code>abstractmethod</code>","text":"<p>Transform a model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>  <code>abstractmethod</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef obtain_parameter_values(\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.seed_model","title":"<code>seed_model(model, seed)</code>  <code>abstractmethod</code>","text":"<p>Set the seed of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef seed_model(self, model: Any, seed: int | None) -&gt; Any:\n    \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Parameters\n    ----------\n    model : Any\n        The model to be seeded\n    seed : int\n\n    Returns\n    -------\n    model\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.get_extension_by_flow","title":"<code>get_extension_by_flow(flow, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented flow. Raises an exception if two extensions can handle a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented flow.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_flow(\n    flow: OpenMLFlow,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented flow.\n    Raises an exception if two extensions can handle a flow.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented flow.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_flow(flow):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle flow: {flow}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle flow: {flow}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.get_extension_by_model","title":"<code>get_extension_by_model(model, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented model. Raises an exception if two extensions can handle a model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented model.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_model(\n    model: Any,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented model.\n    Raises an exception if two extensions can handle a model.\n\n    Parameters\n    ----------\n    model : Any\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented model.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_model(model):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle model: {model}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle model: {model}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.register_extension","title":"<code>register_extension(extension)</code>","text":"<p>Register an extension.</p> <p>Registered extensions are considered by <code>get_extension_by_flow</code> and <code>get_extension_by_model</code>, which are used by <code>openml.flow</code> and <code>openml.runs</code>.</p> <p>Parameters:</p> Name Type Description Default <code>extension</code> <code>Type[Extension]</code> required <p>Returns:</p> Type Description <code>None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def register_extension(extension: type[Extension]) -&gt; None:\n    \"\"\"Register an extension.\n\n    Registered extensions are considered by ``get_extension_by_flow`` and\n    ``get_extension_by_model``, which are used by ``openml.flow`` and ``openml.runs``.\n\n    Parameters\n    ----------\n    extension : Type[Extension]\n\n    Returns\n    -------\n    None\n    \"\"\"\n    openml.extensions.extensions.append(extension)\n</code></pre>"},{"location":"reference/extensions/extension_interface/","title":"extension_interface","text":""},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension","title":"<code>Extension</code>","text":"<p>               Bases: <code>ABC</code></p> <p>Defines the interface to connect machine learning libraries to OpenML-Python.</p> <p>See <code>openml.extension.sklearn.extension</code> for an implementation to bootstrap from.</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>class Extension(ABC):\n    \"\"\"Defines the interface to connect machine learning libraries to OpenML-Python.\n\n    See ``openml.extension.sklearn.extension`` for an implementation to bootstrap from.\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    @abstractmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given flow can be handled by this extension.\n\n        This is typically done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @classmethod\n    @abstractmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model flow can be handled by this extension.\n\n        This is typically done by checking the type of the model, or the package it belongs to.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for flow serialization and de-serialization\n\n    @abstractmethod\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Instantiate a model from the flow representation.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n    @abstractmethod\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n\n    @abstractmethod\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Returns\n        -------\n        List\n        \"\"\"\n\n    @abstractmethod\n    def create_setup_string(self, model: Any) -&gt; str:\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for performing runs with extension modules\n\n    @abstractmethod\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is an estimator for the given extension.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @abstractmethod\n    def seed_model(self, model: Any, seed: int | None) -&gt; Any:\n        \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Parameters\n        ----------\n        model : Any\n            The model to be seeded\n        seed : int\n\n        Returns\n        -------\n        model\n        \"\"\"\n\n    @abstractmethod\n    def _run_model_on_fold(  # noqa: PLR0913\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | None = None,\n    ) -&gt; tuple[np.ndarray, np.ndarray | None, OrderedDict[str, float], OpenMLRunTrace | None]:\n        \"\"\"Run a model on a repeat, fold, subsample triplet of the task.\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        :func:`openml.runs.run_flow_on_task`.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        predictions : np.ndarray\n            Model predictions.\n        probabilities :  Optional, np.ndarray\n            Predicted probabilities (only applicable for supervised classification tasks).\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : Optional, OpenMLRunTrace\n            Hyperparameter optimization trace (only applicable for supervised tasks with\n            hyperparameter optimization).\n        \"\"\"\n\n    @abstractmethod\n    def obtain_parameter_values(\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n\n    @abstractmethod\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for hyperparameter optimization\n\n    @abstractmethod\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n        model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a given flow can be handled by this extension.</p> <p>This is typically done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given flow can be handled by this extension.\n\n    This is typically done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a model flow can be handled by this extension.</p> <p>This is typically done by checking the type of the model, or the package it belongs to.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model flow can be handled by this extension.\n\n    This is typically done by checking the type of the model, or the package it belongs to.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>  <code>abstractmethod</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.create_setup_string","title":"<code>create_setup_string(model)</code>  <code>abstractmethod</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef create_setup_string(self, model: Any) -&gt; str:\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a model from the flow representation.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT002, FBT001\n) -&gt; Any:\n    \"\"\"Instantiate a model from the flow representation.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.get_version_information","title":"<code>get_version_information()</code>  <code>abstractmethod</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Returns\n    -------\n    List\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a base model which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n    model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.is_estimator","title":"<code>is_estimator(model)</code>  <code>abstractmethod</code>","text":"<p>Check whether the given model is an estimator for the given extension.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is an estimator for the given extension.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.model_to_flow","title":"<code>model_to_flow(model)</code>  <code>abstractmethod</code>","text":"<p>Transform a model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>  <code>abstractmethod</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef obtain_parameter_values(\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.seed_model","title":"<code>seed_model(model, seed)</code>  <code>abstractmethod</code>","text":"<p>Set the seed of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef seed_model(self, model: Any, seed: int | None) -&gt; Any:\n    \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Parameters\n    ----------\n    model : Any\n        The model to be seeded\n    seed : int\n\n    Returns\n    -------\n    model\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/functions/","title":"functions","text":""},{"location":"reference/extensions/functions/#openml.extensions.functions.get_extension_by_flow","title":"<code>get_extension_by_flow(flow, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented flow. Raises an exception if two extensions can handle a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented flow.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_flow(\n    flow: OpenMLFlow,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented flow.\n    Raises an exception if two extensions can handle a flow.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented flow.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_flow(flow):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle flow: {flow}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle flow: {flow}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/functions/#openml.extensions.functions.get_extension_by_model","title":"<code>get_extension_by_model(model, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented model. Raises an exception if two extensions can handle a model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented model.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_model(\n    model: Any,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented model.\n    Raises an exception if two extensions can handle a model.\n\n    Parameters\n    ----------\n    model : Any\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented model.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_model(model):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle model: {model}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle model: {model}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/functions/#openml.extensions.functions.register_extension","title":"<code>register_extension(extension)</code>","text":"<p>Register an extension.</p> <p>Registered extensions are considered by <code>get_extension_by_flow</code> and <code>get_extension_by_model</code>, which are used by <code>openml.flow</code> and <code>openml.runs</code>.</p> <p>Parameters:</p> Name Type Description Default <code>extension</code> <code>Type[Extension]</code> required <p>Returns:</p> Type Description <code>None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def register_extension(extension: type[Extension]) -&gt; None:\n    \"\"\"Register an extension.\n\n    Registered extensions are considered by ``get_extension_by_flow`` and\n    ``get_extension_by_model``, which are used by ``openml.flow`` and ``openml.runs``.\n\n    Parameters\n    ----------\n    extension : Type[Extension]\n\n    Returns\n    -------\n    None\n    \"\"\"\n    openml.extensions.extensions.append(extension)\n</code></pre>"},{"location":"reference/extensions/sklearn/","title":"sklearn","text":""},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension","title":"<code>SklearnExtension</code>","text":"<p>               Bases: <code>Extension</code></p> <p>Connect scikit-learn to OpenML-Python. The estimators which use this extension must be scikit-learn compatible, i.e needs to be a subclass of sklearn.base.BaseEstimator\".</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>class SklearnExtension(Extension):\n    \"\"\"Connect scikit-learn to OpenML-Python.\n    The estimators which use this extension must be scikit-learn compatible,\n    i.e needs to be a subclass of sklearn.base.BaseEstimator\".\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given describes a scikit-learn estimator.\n\n        This is done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return cls._is_sklearn_flow(flow)\n\n    @classmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.base.BaseEstimator)\n\n    @classmethod\n    def trim_flow_name(  # noqa: C901\n        cls,\n        long_name: str,\n        extra_trim_length: int = 100,\n        _outer: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; str:\n        \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n        Flows are assumed to have the following naming structure:\n        ``(model_selection)? (pipeline)? (steps)+``\n        and will be shortened to:\n        ``sklearn.(selection.)?(pipeline.)?(steps)+``\n        e.g. (white spaces and newlines added for readability)\n\n        .. code ::\n\n            sklearn.pipeline.Pipeline(\n                columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                    numeric=sklearn.pipeline.Pipeline(\n                        imputer=sklearn.preprocessing.imputation.Imputer,\n                        standardscaler=sklearn.preprocessing.data.StandardScaler),\n                    nominal=sklearn.pipeline.Pipeline(\n                        simpleimputer=sklearn.impute.SimpleImputer,\n                        onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n                variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n                svc=sklearn.svm.classes.SVC)\n\n        -&gt;\n        ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n        Parameters\n        ----------\n        long_name : str\n            The full flow name generated by the scikit-learn extension.\n        extra_trim_length: int (default=100)\n            If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n            of the short name is performed. This reduces the produced short name length.\n            There is no guarantee the end result will not exceed `extra_trim_length`.\n        _outer : bool (default=True)\n            For internal use only. Specifies if the function is called recursively.\n\n        Returns\n        -------\n        str\n\n        \"\"\"\n\n        def remove_all_in_parentheses(string: str) -&gt; str:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            while removals &gt; 0:\n                string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            return string\n\n        # Generally, we want to trim all hyperparameters, the exception to that is for model\n        # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n        # So we first trim name of the `estimator` specified in mode selection. For reference, in\n        # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n        # keep it in the final trimmed flow name:\n        # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n        # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n        # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n        # sklearn.tree.tree.DecisionTreeClassifier))\n        if \"sklearn.model_selection\" in long_name:\n            start_index = long_name.index(\"sklearn.model_selection\")\n            estimator_start = (\n                start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n            )\n\n            model_select_boilerplate = long_name[start_index:estimator_start]\n            # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n            model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n            # Now we want to also find and parse the `estimator`, for this we find the closing\n            # parenthesis to the model selection technique:\n            closing_parenthesis_expected = 1\n            for char in long_name[estimator_start:]:\n                if char == \"(\":\n                    closing_parenthesis_expected += 1\n                if char == \")\":\n                    closing_parenthesis_expected -= 1\n                if closing_parenthesis_expected == 0:\n                    break\n\n            _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n            model_select_pipeline = long_name[estimator_start:_end]\n\n            trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n            _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n            model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n            name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n        else:\n            name = long_name\n\n        module_name = long_name.split(\".\")[0]\n        short_name = module_name + \".{}\"\n\n        if name.startswith(\"sklearn.pipeline\"):\n            full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n            pipeline_class = full_pipeline_class.split(\".\")[-1]\n            # We don't want nested pipelines in the short name, so we trim all complicated\n            # subcomponents, i.e. those with parentheses:\n            pipeline = remove_all_in_parentheses(pipeline)\n\n            # then the pipeline steps are formatted e.g.:\n            # step1name=sklearn.submodule.ClassName,step2name...\n            components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n            pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n            if len(short_name.format(pipeline)) &gt; extra_trim_length:\n                pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n        else:\n            # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n            pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n        if not _outer:\n            # Anything from parenthesis in inner calls should not be culled, so we use brackets\n            pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n        else:\n            # Square brackets may be introduced with nested model_selection\n            pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n        return short_name.format(pipeline)\n\n    @classmethod\n    def _min_dependency_str(cls, sklearn_version: str) -&gt; str:\n        \"\"\"Returns a string containing the minimum dependencies for the sklearn version passed.\n\n        Parameters\n        ----------\n        sklearn_version : str\n            A version string of the xx.xx.xx\n\n        Returns\n        -------\n        str\n        \"\"\"\n        openml_major_version = int(LooseVersion(openml.__version__).version[1])\n        # This explicit check is necessary to support existing entities on the OpenML servers\n        # that used the fixed dependency string (in the else block)\n        if openml_major_version &gt; 11:\n            # OpenML v0.11 onwards supports sklearn&gt;=0.24\n            # assumption: 0.24 onwards sklearn should contain a _min_dependencies.py file with\n            # variables declared for extracting minimum dependency for that version\n            if LooseVersion(sklearn_version) &gt;= \"0.24\":\n                from sklearn import _min_dependencies as _mindep\n\n                dependency_list = {\n                    \"numpy\": f\"{_mindep.NUMPY_MIN_VERSION}\",\n                    \"scipy\": f\"{_mindep.SCIPY_MIN_VERSION}\",\n                    \"joblib\": f\"{_mindep.JOBLIB_MIN_VERSION}\",\n                    \"threadpoolctl\": f\"{_mindep.THREADPOOLCTL_MIN_VERSION}\",\n                }\n            elif LooseVersion(sklearn_version) &gt;= \"0.23\":\n                dependency_list = {\n                    \"numpy\": \"1.13.3\",\n                    \"scipy\": \"0.19.1\",\n                    \"joblib\": \"0.11\",\n                    \"threadpoolctl\": \"2.0.0\",\n                }\n                if LooseVersion(sklearn_version).version[2] == 0:\n                    dependency_list.pop(\"threadpoolctl\")\n            elif LooseVersion(sklearn_version) &gt;= \"0.21\":\n                dependency_list = {\"numpy\": \"1.11.0\", \"scipy\": \"0.17.0\", \"joblib\": \"0.11\"}\n            elif LooseVersion(sklearn_version) &gt;= \"0.19\":\n                dependency_list = {\"numpy\": \"1.8.2\", \"scipy\": \"0.13.3\"}\n            else:\n                dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n        else:\n            # this is INCORRECT for sklearn versions &gt;= 0.19 and &lt; 0.24\n            # given that OpenML has existing flows uploaded with such dependency information,\n            # we change no behaviour for older sklearn version, however from 0.24 onwards\n            # the dependency list will be accurately updated for any flow uploaded to OpenML\n            dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n\n        sklearn_dep = f\"sklearn=={sklearn_version}\"\n        dep_str = \"\\n\".join([f\"{k}&gt;={v}\" for k, v in dependency_list.items()])\n        return \"\\n\".join([sklearn_dep, dep_str])\n\n    ################################################################################################\n    # Methods for flow serialization and de-serialization\n\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; Any:\n        \"\"\"Initializes a sklearn model based on a flow.\n\n        Parameters\n        ----------\n        flow : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        return self._deserialize_sklearn(\n            flow,\n            initialize_with_defaults=initialize_with_defaults,\n            strict_version=strict_version,\n        )\n\n    def _deserialize_sklearn(  # noqa: PLR0915, C901, PLR0913, PLR0912\n        self,\n        o: Any,\n        components: dict | None = None,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        recursion_depth: int = 0,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Recursive function to deserialize a scikit-learn flow.\n\n        This function inspects an object to deserialize and decides how to do so. This function\n        delegates all work to the respective functions to deserialize special data structures etc.\n        This function works on everything that has been serialized to OpenML: OpenMLFlow,\n        components (which are flows themselves), functions, hyperparameter distributions (for\n        random search) and the actual hyperparameter values themselves.\n\n        Parameters\n        ----------\n        o : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        components : Optional[dict]\n            Components of the current flow being de-serialized. These will not be used when\n            de-serializing the actual flow, but when de-serializing a component reference.\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        recursion_depth : int\n            The depth at which this flow is called, mostly for debugging\n            purposes\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        logger.info(\n            \"-{} flow_to_sklearn START o={}, components={}, init_defaults={}\".format(\n                \"-\" * recursion_depth, o, components, initialize_with_defaults\n            ),\n        )\n        depth_pp = recursion_depth + 1  # shortcut var, depth plus plus\n\n        # First, we need to check whether the presented object is a json string.\n        # JSON strings are used to encoder parameter values. By passing around\n        # json strings for parameters, we make sure that we can flow_to_sklearn\n        # the parameter values to the correct type.\n\n        if isinstance(o, str):\n            with contextlib.suppress(JSONDecodeError):\n                o = json.loads(o)\n\n        if isinstance(o, dict):\n            # Check if the dict encodes a 'special' object, which could not\n            # easily converted into a string, but rather the information to\n            # re-create the object were stored in a dictionary.\n            if \"oml-python:serialized_object\" in o:\n                serialized_type = o[\"oml-python:serialized_object\"]\n                value = o[\"value\"]\n                if serialized_type == \"type\":\n                    rval = self._deserialize_type(value)\n                elif serialized_type == \"rv_frozen\":\n                    rval = self._deserialize_rv_frozen(value)\n                elif serialized_type == \"function\":\n                    rval = self._deserialize_function(value)\n                elif serialized_type in (COMPOSITION_STEP_CONSTANT, COMPONENT_REFERENCE):\n                    if serialized_type == COMPOSITION_STEP_CONSTANT:\n                        pass\n                    elif serialized_type == COMPONENT_REFERENCE:\n                        value = self._deserialize_sklearn(\n                            value,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        )\n                    else:\n                        raise NotImplementedError(serialized_type)\n                    assert components is not None  # Necessary for mypy\n                    step_name = value[\"step_name\"]\n                    key = value[\"key\"]\n                    component = self._deserialize_sklearn(\n                        components[key],\n                        initialize_with_defaults=initialize_with_defaults,\n                        recursion_depth=depth_pp,\n                        strict_version=strict_version,\n                    )\n                    # The component is now added to where it should be used\n                    # later. It should not be passed to the constructor of the\n                    # main flow object.\n                    del components[key]\n                    if step_name is None:\n                        rval = component\n                    elif \"argument_1\" not in value:\n                        rval = (step_name, component)\n                    else:\n                        rval = (step_name, component, value[\"argument_1\"])\n                elif serialized_type == \"cv_object\":\n                    rval = self._deserialize_cross_validator(\n                        value,\n                        recursion_depth=recursion_depth,\n                        strict_version=strict_version,\n                    )\n                else:\n                    raise ValueError(\"Cannot flow_to_sklearn %s\" % serialized_type)\n\n            else:\n                rval = OrderedDict(\n                    (\n                        self._deserialize_sklearn(\n                            o=key,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                        self._deserialize_sklearn(\n                            o=value,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                    )\n                    for key, value in sorted(o.items())\n                )\n        elif isinstance(o, (list, tuple)):\n            rval = [\n                self._deserialize_sklearn(\n                    o=element,\n                    components=components,\n                    initialize_with_defaults=initialize_with_defaults,\n                    recursion_depth=depth_pp,\n                    strict_version=strict_version,\n                )\n                for element in o\n            ]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, (bool, int, float, str)) or o is None:\n            rval = o\n        elif isinstance(o, OpenMLFlow):\n            if not self._is_sklearn_flow(o):\n                raise ValueError(\"Only sklearn flows can be reinstantiated\")\n            rval = self._deserialize_model(\n                flow=o,\n                keep_defaults=initialize_with_defaults,\n                recursion_depth=recursion_depth,\n                strict_version=strict_version,\n            )\n        else:\n            raise TypeError(o)\n        logger.info(\n            \"-{} flow_to_sklearn END   o={}, rval={}\".format(\"-\" * recursion_depth, o, rval)\n        )\n        return rval\n\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n        # Necessary to make pypy not complain about all the different possible return types\n        return self._serialize_sklearn(model)\n\n    def _serialize_sklearn(self, o: Any, parent_model: Any | None = None) -&gt; Any:  # noqa: PLR0912, C901\n        rval = None  # type: Any\n\n        # TODO: assert that only on first recursion lvl `parent_model` can be None\n        if self.is_estimator(o):\n            # is the main model or a submodel\n            rval = self._serialize_model(o)\n        elif (\n            isinstance(o, (list, tuple))\n            and len(o) == 2\n            and o[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            and isinstance(parent_model, sklearn.pipeline._BaseComposition)\n        ):\n            rval = o\n        elif isinstance(o, (list, tuple)):\n            # TODO: explain what type of parameter is here\n            rval = [self._serialize_sklearn(element, parent_model) for element in o]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, SIMPLE_TYPES) or o is None:\n            if isinstance(o, tuple(SIMPLE_NUMPY_TYPES)):\n                o = o.item()  # type: ignore\n            # base parameter values\n            rval = o\n        elif isinstance(o, dict):\n            # TODO: explain what type of parameter is here\n            if not isinstance(o, OrderedDict):\n                o = OrderedDict(sorted(o.items()))\n\n            rval = OrderedDict()\n            for key, value in o.items():\n                if not isinstance(key, str):\n                    raise TypeError(\n                        \"Can only use string as keys, you passed \"\n                        f\"type {type(key)} for value {key!s}.\",\n                    )\n                _key = self._serialize_sklearn(key, parent_model)\n                rval[_key] = self._serialize_sklearn(value, parent_model)\n        elif isinstance(o, type):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_type(o)\n        elif isinstance(o, scipy.stats.distributions.rv_frozen):\n            rval = self._serialize_rv_frozen(o)\n        # This only works for user-defined functions (and not even partial).\n        # I think this is exactly what we want here as there shouldn't be any\n        # built-in or functool.partials in a pipeline\n        elif inspect.isfunction(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_function(o)\n        elif self._is_cross_validator(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_cross_validator(o)\n        else:\n            raise TypeError(o, type(o))\n\n        return rval\n\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n        Returns\n        -------\n        List\n        \"\"\"\n        # This can possibly be done by a package such as pyxb, but I could not get\n        # it to work properly.\n        import numpy\n        import scipy\n        import sklearn\n\n        major, minor, micro, _, _ = sys.version_info\n        python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n        sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n        numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n        scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n        return [python_version, sklearn_version, numpy_version, scipy_version]\n\n    def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return \" \".join(self.get_version_information())\n\n    def _is_cross_validator(self, o: Any) -&gt; bool:\n        return isinstance(o, sklearn.model_selection.BaseCrossValidator)\n\n    @classmethod\n    def _is_sklearn_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        sklearn_dependency = isinstance(flow.dependencies, str) and \"sklearn\" in flow.dependencies\n        sklearn_as_external = isinstance(flow.external_version, str) and (\n            flow.external_version.startswith(\"sklearn==\") or \",sklearn==\" in flow.external_version\n        )\n        return sklearn_dependency or sklearn_as_external\n\n    def _get_sklearn_description(self, model: Any, char_lim: int = 1024) -&gt; str:\n        r\"\"\"Fetches the sklearn function docstring for the flow description\n\n        Retrieves the sklearn docstring available and does the following:\n        * If length of docstring &lt;= char_lim, then returns the complete docstring\n        * Else, trims the docstring till it encounters a 'Read more in the :ref:'\n        * Or till it encounters a 'Parameters\\n----------\\n'\n        The final string returned is at most of length char_lim with leading and\n        trailing whitespaces removed.\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters for the 'description' field.\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return \"\"\n        try:\n            # trim till 'Read more'\n            pattern = \"Read more in the :ref:\"\n            index = s.index(pattern)\n            s = s[:index]\n            # trimming docstring to be within char_lim\n            if len(s) &gt; char_lim:\n                s = f\"{s[: char_lim - 3]}...\"\n            return s.strip()\n        except ValueError:\n            logger.warning(\n                \"'Read more' not found in descriptions. \"\n                \"Trying to trim till 'Parameters' if available in docstring.\",\n            )\n        try:\n            # if 'Read more' doesn't exist, trim till 'Parameters'\n            pattern = \"Parameters\"\n            index = s.index(match_format(pattern))\n        except ValueError:\n            # returning full docstring\n            logger.warning(\"'Parameters' not found in docstring. Omitting docstring trimming.\")\n            index = len(s)\n        s = s[:index]\n        # trimming docstring to be within char_lim\n        if len(s) &gt; char_lim:\n            s = f\"{s[: char_lim - 3]}...\"\n        return s.strip()\n\n    def _extract_sklearn_parameter_docstring(self, model) -&gt; None | str:\n        \"\"\"Extracts the part of sklearn docstring containing parameter information\n\n        Fetches the entire docstring and trims just the Parameter section.\n        The assumption is that 'Parameters' is the first section in sklearn docstrings,\n        followed by other sections titled 'Attributes', 'See also', 'Note', 'References',\n        appearing in that order if defined.\n        Returns a None if no section with 'Parameters' can be found in the docstring.\n\n        Parameters\n        ----------\n        model : sklearn model\n\n        Returns\n        -------\n        str, or None\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return None\n        try:\n            index1 = s.index(match_format(\"Parameters\"))\n        except ValueError as e:\n            # when sklearn docstring has no 'Parameters' section\n            logger.warning(\"{} {}\".format(match_format(\"Parameters\"), e))\n            return None\n\n        headings = [\"Attributes\", \"Notes\", \"See also\", \"Note\", \"References\"]\n        for h in headings:\n            try:\n                # to find end of Parameters section\n                index2 = s.index(match_format(h))\n                break\n            except ValueError:\n                logger.warning(f\"{h} not available in docstring\")\n                continue\n        else:\n            # in the case only 'Parameters' exist, trim till end of docstring\n            index2 = len(s)\n        s = s[index1:index2]\n        return s.strip()\n\n    def _extract_sklearn_param_info(self, model, char_lim=1024) -&gt; None | dict:\n        \"\"\"Parses parameter type and description from sklearn dosctring\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters string fields.\n\n        Returns\n        -------\n        Dict, or None\n        \"\"\"\n        docstring = self._extract_sklearn_parameter_docstring(model)\n        if docstring is None:\n            # when sklearn docstring has no 'Parameters' section\n            return None\n\n        n = re.compile(\"[.]*\\n\", flags=IGNORECASE)\n        lines = n.split(docstring)\n        p = re.compile(\"[a-z0-9_ ]+ : [a-z0-9_']+[a-z0-9_ ]*\", flags=IGNORECASE)\n        # The above regular expression is designed to detect sklearn parameter names and type\n        # in the format of [variable_name][space]:[space][type]\n        # The expectation is that the parameter description for this detected parameter will\n        # be all the lines in the docstring till the regex finds another parameter match\n\n        # collecting parameters and their descriptions\n        description = []  # type: List\n        for s in lines:\n            param = p.findall(s)\n            if param != []:\n                # a parameter definition is found by regex\n                # creating placeholder when parameter found which will be a list of strings\n                # string descriptions will be appended in subsequent iterations\n                # till another parameter is found and a new placeholder is created\n                placeholder = [\"\"]  # type: List[str]\n                description.append(placeholder)\n            elif len(description) &gt; 0:  # description=[] means no parameters found yet\n                # appending strings to the placeholder created when parameter found\n                description[-1].append(s)\n        for i in range(len(description)):\n            # concatenating parameter description strings\n            description[i] = \"\\n\".join(description[i]).strip()\n            # limiting all parameter descriptions to accepted OpenML string length\n            if len(description[i]) &gt; char_lim:\n                description[i] = f\"{description[i][: char_lim - 3]}...\"\n\n        # collecting parameters and their types\n        parameter_docs = OrderedDict()\n        matches = p.findall(docstring)\n        for i, param in enumerate(matches):\n            key, value = str(param).split(\":\")\n            parameter_docs[key.strip()] = [value.strip(), description[i]]\n\n        # to avoid KeyError for missing parameters\n        param_list_true = list(model.get_params().keys())\n        param_list_found = list(parameter_docs.keys())\n        for param in list(set(param_list_true) - set(param_list_found)):\n            parameter_docs[param] = [None, None]\n\n        return parameter_docs\n\n    def _serialize_model(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Create an OpenMLFlow.\n\n        Calls `sklearn_to_flow` recursively to properly serialize the\n        parameters to strings and the components (other models) to OpenMLFlows.\n\n        Parameters\n        ----------\n        model : sklearn estimator\n\n        Returns\n        -------\n        OpenMLFlow\n\n        \"\"\"\n        # Get all necessary information about the model objects itself\n        (\n            parameters,\n            parameters_meta_info,\n            subcomponents,\n            subcomponents_explicit,\n        ) = self._extract_information_from_model(model)\n\n        # Check that a component does not occur multiple times in a flow as this\n        # is not supported by OpenML\n        self._check_multiple_occurence_of_component_in_flow(model, subcomponents)\n\n        # Create a flow name, which contains all components in brackets, e.g.:\n        # RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),\n        # StandardScaler,AdaBoostClassifier(DecisionTreeClassifier))\n        class_name = model.__module__ + \".\" + model.__class__.__name__\n\n        # will be part of the name (in brackets)\n        sub_components_names = \"\"\n        for key in subcomponents:\n            name_thing = subcomponents[key]\n            if isinstance(name_thing, OpenMLFlow):\n                name = name_thing.name\n            elif (\n                isinstance(name_thing, str)\n                and subcomponents[key] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                name = name_thing\n            else:\n                raise TypeError(type(subcomponents[key]))\n\n            if key in subcomponents_explicit:\n                sub_components_names += \",\" + key + \"=\" + name\n            else:\n                sub_components_names += \",\" + name\n\n        # slice operation on string in order to get rid of leading comma\n        name = f\"{class_name}({sub_components_names[1:]})\" if sub_components_names else class_name\n        short_name = SklearnExtension.trim_flow_name(name)\n\n        # Get the external versions of all sub-components\n        external_version = self._get_external_version_string(model, subcomponents)\n        dependencies = self._get_dependencies()\n        tags = self._get_tags()\n\n        sklearn_description = self._get_sklearn_description(model)\n        return OpenMLFlow(\n            name=name,\n            class_name=class_name,\n            custom_name=short_name,\n            description=sklearn_description,\n            model=model,\n            components=subcomponents,\n            parameters=parameters,\n            parameters_meta_info=parameters_meta_info,\n            external_version=external_version,\n            tags=tags,\n            extension=self,\n            language=\"English\",\n            dependencies=dependencies,\n        )\n\n    def _get_dependencies(self) -&gt; str:\n        return self._min_dependency_str(sklearn.__version__)  # type: ignore\n\n    def _get_tags(self) -&gt; list[str]:\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        sklearn_version_formatted = sklearn_version.replace(\"==\", \"_\")\n        return [\n            \"openml-python\",\n            \"sklearn\",\n            \"scikit-learn\",\n            \"python\",\n            sklearn_version_formatted,\n            # TODO: add more tags based on the scikit-learn\n            # module a flow is in? For example automatically\n            # annotate a class of sklearn.svm.SVC() with the\n            # tag svm?\n        ]\n\n    def _get_external_version_string(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; str:\n        # Create external version string for a flow, given the model and the\n        # already parsed dictionary of sub_components. Retrieves the external\n        # version of all subcomponents, which themselves already contain all\n        # requirements for their subcomponents. The external version string is a\n        # sorted concatenation of all modules which are present in this run.\n\n        external_versions = set()\n\n        # The model is None if the flow is a placeholder flow such as 'passthrough' or 'drop'\n        if model is not None:\n            model_package_name = model.__module__.split(\".\")[0]\n            module = importlib.import_module(model_package_name)\n            model_package_version_number = module.__version__  # type: ignore\n            external_version = self._format_external_version(\n                model_package_name,\n                model_package_version_number,\n            )\n            external_versions.add(external_version)\n\n        openml_version = self._format_external_version(\"openml\", openml.__version__)\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        external_versions.add(openml_version)\n        external_versions.add(sklearn_version)\n        for visitee in sub_components.values():\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                continue\n            for external_version in visitee.external_version.split(\",\"):\n                external_versions.add(external_version)\n        return \",\".join(sorted(external_versions))\n\n    def _check_multiple_occurence_of_component_in_flow(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; None:\n        to_visit_stack: list[OpenMLFlow] = []\n        to_visit_stack.extend(sub_components.values())\n        known_sub_components: set[str] = set()\n\n        while len(to_visit_stack) &gt; 0:\n            visitee = to_visit_stack.pop()\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                known_sub_components.add(visitee)\n            elif visitee.name in known_sub_components:\n                raise ValueError(\n                    f\"Found a second occurence of component {visitee.name} when \"\n                    f\"trying to serialize {model}.\",\n                )\n            else:\n                known_sub_components.add(visitee.name)\n                to_visit_stack.extend(visitee.components.values())\n\n    def _extract_information_from_model(  # noqa: PLR0915, C901, PLR0912\n        self,\n        model: Any,\n    ) -&gt; tuple[\n        OrderedDict[str, str | None],\n        OrderedDict[str, dict | None],\n        OrderedDict[str, OpenMLFlow],\n        set,\n    ]:\n        # This function contains four \"global\" states and is quite long and\n        # complicated. If it gets to complicated to ensure it's correctness,\n        # it would be best to make it a class with the four \"global\" states being\n        # the class attributes and the if/elif/else in the for-loop calls to\n        # separate class methods\n\n        # stores all entities that should become subcomponents\n        sub_components = OrderedDict()  # type: OrderedDict[str, OpenMLFlow]\n        # stores the keys of all subcomponents that should become\n        sub_components_explicit = set()\n        parameters: OrderedDict[str, str | None] = OrderedDict()\n        parameters_meta_info: OrderedDict[str, dict | None] = OrderedDict()\n        parameters_docs = self._extract_sklearn_param_info(model)\n\n        model_parameters = model.get_params(deep=False)\n        for k, v in sorted(model_parameters.items(), key=lambda t: t[0]):\n            rval = self._serialize_sklearn(v, model)\n\n            def flatten_all(list_):\n                \"\"\"Flattens arbitrary depth lists of lists (e.g. [[1,2],[3,[1]]] -&gt; [1,2,3,1]).\"\"\"\n                for el in list_:\n                    if isinstance(el, (list, tuple)) and len(el) &gt; 0:\n                        yield from flatten_all(el)\n                    else:\n                        yield el\n\n            # In case rval is a list of lists (or tuples), we need to identify two situations:\n            # - sklearn pipeline steps, feature union or base classifiers in voting classifier.\n            #   They look like e.g. [(\"imputer\", Imputer()), (\"classifier\", SVC())]\n            # - a list of lists with simple types (e.g. int or str), such as for an OrdinalEncoder\n            #   where all possible values for each feature are described: [[0,1,2], [1,2,5]]\n            is_non_empty_list_of_lists_with_same_type = (\n                isinstance(rval, (list, tuple))\n                and len(rval) &gt; 0\n                and isinstance(rval[0], (list, tuple))\n                and all(isinstance(rval_i, type(rval[0])) for rval_i in rval)\n            )\n\n            # Check that all list elements are of simple types.\n            nested_list_of_simple_types = (\n                is_non_empty_list_of_lists_with_same_type\n                and all(isinstance(el, SIMPLE_TYPES) for el in flatten_all(rval))\n                and all(\n                    len(rv) in (2, 3) and rv[1] not in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    for rv in rval\n                )\n            )\n\n            if is_non_empty_list_of_lists_with_same_type and not nested_list_of_simple_types:\n                # If a list of lists is identified that include 'non-simple' types (e.g. objects),\n                # we assume they are steps in a pipeline, feature union, or base classifiers in\n                # a voting classifier.\n                parameter_value = []  # type: List\n                reserved_keywords = set(model.get_params(deep=False).keys())\n\n                for sub_component_tuple in rval:\n                    identifier = sub_component_tuple[0]\n                    sub_component = sub_component_tuple[1]\n                    sub_component_type = type(sub_component_tuple)\n                    if not 2 &lt;= len(sub_component_tuple) &lt;= 3:\n                        # length 2 is for {VotingClassifier.estimators,\n                        # Pipeline.steps, FeatureUnion.transformer_list}\n                        # length 3 is for ColumnTransformer\n                        msg = \"Length of tuple of type {} does not match assumptions\".format(\n                            sub_component_type,\n                        )\n                        raise ValueError(msg)\n\n                    if isinstance(sub_component, str):\n                        if sub_component not in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                            msg = (\n                                \"Second item of tuple does not match assumptions. \"\n                                \"If string, can be only 'drop' or 'passthrough' but\"\n                                \"got %s\" % sub_component\n                            )\n                            raise ValueError(msg)\n                    elif sub_component is None:\n                        msg = (\n                            \"Cannot serialize objects of None type. Please use a valid \"\n                            \"placeholder for None. Note that empty sklearn estimators can be \"\n                            \"replaced with 'drop' or 'passthrough'.\"\n                        )\n                        raise ValueError(msg)\n                    elif not isinstance(sub_component, OpenMLFlow):\n                        msg = (\n                            \"Second item of tuple does not match assumptions. \"\n                            \"Expected OpenMLFlow, got %s\" % type(sub_component)\n                        )\n                        raise TypeError(msg)\n\n                    if identifier in reserved_keywords:\n                        parent_model = f\"{model.__module__}.{model.__class__.__name__}\"\n                        msg = \"Found element shadowing official \" \"parameter for {}: {}\".format(\n                            parent_model,\n                            identifier,\n                        )\n                        raise PyOpenMLError(msg)\n\n                    # when deserializing the parameter\n                    sub_components_explicit.add(identifier)\n                    if isinstance(sub_component, str):\n                        external_version = self._get_external_version_string(None, {})\n                        dependencies = self._get_dependencies()\n                        tags = self._get_tags()\n\n                        sub_components[identifier] = OpenMLFlow(\n                            name=sub_component,\n                            description=\"Placeholder flow for scikit-learn's string pipeline \"\n                            \"members\",\n                            components=OrderedDict(),\n                            parameters=OrderedDict(),\n                            parameters_meta_info=OrderedDict(),\n                            external_version=external_version,\n                            tags=tags,\n                            language=\"English\",\n                            dependencies=dependencies,\n                            model=None,\n                        )\n                        component_reference: OrderedDict[str, str | dict] = OrderedDict()\n                        component_reference[\n                            \"oml-python:serialized_object\"\n                        ] = COMPOSITION_STEP_CONSTANT\n                        cr_value: dict[str, Any] = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    else:\n                        sub_components[identifier] = sub_component\n                        component_reference = OrderedDict()\n                        component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                        cr_value = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    parameter_value.append(component_reference)\n\n                # Here (and in the elif and else branch below) are the only\n                # places where we encode a value as json to make sure that all\n                # parameter values still have the same type after\n                # deserialization\n                if isinstance(rval, tuple):\n                    parameter_json = json.dumps(tuple(parameter_value))\n                else:\n                    parameter_json = json.dumps(parameter_value)\n                parameters[k] = parameter_json\n\n            elif isinstance(rval, OpenMLFlow):\n                # A subcomponent, for example the base model in\n                # AdaBoostClassifier\n                sub_components[k] = rval\n                sub_components_explicit.add(k)\n                component_reference = OrderedDict()\n                component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                cr_value = OrderedDict()\n                cr_value[\"key\"] = k\n                cr_value[\"step_name\"] = None\n                component_reference[\"value\"] = cr_value\n                cr = self._serialize_sklearn(component_reference, model)\n                parameters[k] = json.dumps(cr)\n\n            elif not (hasattr(rval, \"__len__\") and len(rval) == 0):\n                rval = json.dumps(rval)\n                parameters[k] = rval\n            # a regular hyperparameter\n            else:\n                parameters[k] = None\n\n            if parameters_docs is not None:\n                data_type, description = parameters_docs[k]\n                parameters_meta_info[k] = OrderedDict(\n                    ((\"description\", description), (\"data_type\", data_type)),\n                )\n            else:\n                parameters_meta_info[k] = OrderedDict(((\"description\", None), (\"data_type\", None)))\n\n        return parameters, parameters_meta_info, sub_components, sub_components_explicit\n\n    def _get_fn_arguments_with_defaults(self, fn_name: Callable) -&gt; tuple[dict, set]:\n        \"\"\"\n        Returns\n        -------\n            i) a dict with all parameter names that have a default value, and\n            ii) a set with all parameter names that do not have a default\n\n        Parameters\n        ----------\n        fn_name : callable\n            The function of which we want to obtain the defaults\n\n        Returns\n        -------\n        params_with_defaults: dict\n            a dict mapping parameter name to the default value\n        params_without_defaults: set\n            a set with all parameters that do not have a default value\n        \"\"\"\n        # parameters with defaults are optional, all others are required.\n        parameters = inspect.signature(fn_name).parameters\n        required_params = set()\n        optional_params = {}\n        for param in parameters:\n            parameter = parameters.get(param)\n            default_val = parameter.default  # type: ignore\n            if default_val is inspect.Signature.empty:\n                required_params.add(param)\n            else:\n                optional_params[param] = default_val\n        return optional_params, required_params\n\n    def _deserialize_model(\n        self,\n        flow: OpenMLFlow,\n        keep_defaults: bool,  # noqa: FBT001\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        logger.info(\"-{} deserialize {}\".format(\"-\" * recursion_depth, flow.name))\n        model_name = flow.class_name\n        self._check_dependencies(flow.dependencies, strict_version=strict_version)\n\n        parameters = flow.parameters\n        components = flow.components\n        parameter_dict: dict[str, Any] = OrderedDict()\n\n        # Do a shallow copy of the components dictionary so we can remove the\n        # components from this copy once we added them into the pipeline. This\n        # allows us to not consider them any more when looping over the\n        # components, but keeping the dictionary of components untouched in the\n        # original components dictionary.\n        components_ = copy.copy(components)\n\n        for name in parameters:\n            value = parameters.get(name)\n            logger.info(\n                \"--{} flow_parameter={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                components=components_,\n                initialize_with_defaults=keep_defaults,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        for name in components:\n            if name in parameter_dict:\n                continue\n            if name not in components_:\n                continue\n            value = components[name]\n            logger.info(\n                \"--{} flow_component={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        if model_name is None and flow.name in SKLEARN_PIPELINE_STRING_COMPONENTS:\n            return flow.name\n\n        assert model_name is not None\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n\n        if keep_defaults:\n            # obtain all params with a default\n            param_defaults, _ = self._get_fn_arguments_with_defaults(model_class.__init__)\n\n            # delete the params that have a default from the dict,\n            # so they get initialized with their default value\n            # except [...]\n            for param in param_defaults:\n                # [...] the ones that also have a key in the components dict.\n                # As OpenML stores different flows for ensembles with different\n                # (base-)components, in OpenML terms, these are not considered\n                # hyperparameters but rather constants (i.e., changing them would\n                # result in a different flow)\n                if param not in components:\n                    del parameter_dict[param]\n\n        return model_class(**parameter_dict)\n\n    def _check_dependencies(\n        self,\n        dependencies: str,\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        if not dependencies:\n            return\n\n        dependencies_list = dependencies.split(\"\\n\")\n        for dependency_string in dependencies_list:\n            match = DEPENDENCIES_PATTERN.match(dependency_string)\n            if not match:\n                raise ValueError(\"Cannot parse dependency %s\" % dependency_string)\n\n            dependency_name = match.group(\"name\")\n            operation = match.group(\"operation\")\n            version = match.group(\"version\")\n\n            module = importlib.import_module(dependency_name)\n            required_version = LooseVersion(version)\n            installed_version = LooseVersion(module.__version__)  # type: ignore\n\n            if operation == \"==\":\n                check = required_version == installed_version\n            elif operation == \"&gt;\":\n                check = installed_version &gt; required_version\n            elif operation == \"&gt;=\":\n                check = (\n                    installed_version &gt; required_version or installed_version == required_version\n                )\n            else:\n                raise NotImplementedError(\"operation '%s' is not supported\" % operation)\n            message = (\n                \"Trying to deserialize a model with dependency \"\n                f\"{dependency_string} not satisfied.\"\n            )\n            if not check:\n                if strict_version:\n                    raise ValueError(message)\n\n                warnings.warn(message, category=UserWarning, stacklevel=2)\n\n    def _serialize_type(self, o: Any) -&gt; OrderedDict[str, str]:\n        mapping = {\n            float: \"float\",\n            np.float32: \"np.float32\",\n            np.float64: \"np.float64\",\n            int: \"int\",\n            np.int32: \"np.int32\",\n            np.int64: \"np.int64\",\n        }\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[float] = \"np.float\"\n            mapping[int] = \"np.int\"\n\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"type\"\n        ret[\"value\"] = mapping[o]\n        return ret\n\n    def _deserialize_type(self, o: str) -&gt; Any:\n        mapping = {\n            \"float\": float,\n            \"np.float32\": np.float32,\n            \"np.float64\": np.float64,\n            \"int\": int,\n            \"np.int32\": np.int32,\n            \"np.int64\": np.int64,\n        }\n\n        # TODO(eddiebergman): Might be able to remove this\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[\"np.float\"] = np.float  # type: ignore # noqa: NPY001\n            mapping[\"np.int\"] = np.int  # type: ignore # noqa: NPY001\n\n        return mapping[o]\n\n    def _serialize_rv_frozen(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        args = o.args\n        kwds = o.kwds\n        a = o.a\n        b = o.b\n        dist = o.dist.__class__.__module__ + \".\" + o.dist.__class__.__name__\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n        ret[\"oml-python:serialized_object\"] = \"rv_frozen\"\n        ret[\"value\"] = OrderedDict(\n            ((\"dist\", dist), (\"a\", a), (\"b\", b), (\"args\", args), (\"kwds\", kwds)),\n        )\n        return ret\n\n    def _deserialize_rv_frozen(self, o: OrderedDict[str, str]) -&gt; Any:\n        args = o[\"args\"]\n        kwds = o[\"kwds\"]\n        a = o[\"a\"]\n        b = o[\"b\"]\n        dist_name = o[\"dist\"]\n\n        module_name = dist_name.rsplit(\".\", 1)\n        try:\n            rv_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        except AttributeError as e:\n            _tb = traceback.format_exc()\n            warnings.warn(\n                f\"Cannot create model {dist_name} for flow. Reason is from error {type(e)}:{e}\"\n                f\"\\nTraceback: {_tb}\",\n                RuntimeWarning,\n                stacklevel=2,\n            )\n            return None\n\n        dist = scipy.stats.distributions.rv_frozen(rv_class(), *args, **kwds)  # type: ignore\n        dist.a = a\n        dist.b = b\n\n        return dist\n\n    def _serialize_function(self, o: Callable) -&gt; OrderedDict[str, str]:\n        name = o.__module__ + \".\" + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"function\"\n        ret[\"value\"] = name\n        return ret\n\n    def _deserialize_function(self, name: str) -&gt; Callable:\n        module_name = name.rsplit(\".\", 1)\n        return getattr(importlib.import_module(module_name[0]), module_name[1])\n\n    def _serialize_cross_validator(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n\n        parameters = OrderedDict()  # type: 'OrderedDict[str, Any]'\n\n        # XXX this is copied from sklearn.model_selection._split\n        cls = o.__class__\n        init = getattr(cls.__init__, \"deprecated_original\", cls.__init__)\n        # Ignore varargs, kw and default values and pop self\n        init_signature = inspect.signature(init)  # type: ignore\n        # Consider the constructor parameters excluding 'self'\n        if init is object.__init__:\n            args = []  # type: List\n        else:\n            args = sorted(\n                [\n                    p.name\n                    for p in init_signature.parameters.values()\n                    if p.name != \"self\" and p.kind != p.VAR_KEYWORD\n                ],\n            )\n\n        for key in args:\n            # We need deprecation warnings to always be on in order to\n            # catch deprecated param values.\n            # This is set in utils/__init__.py but it gets overwritten\n            # when running under python3 somehow.\n            with warnings.catch_warnings(record=True) as w:\n                warnings.simplefilter(\"always\", DeprecationWarning)\n                value = getattr(o, key, None)\n                if w is not None and len(w) and w[0].category == DeprecationWarning:\n                    # if the parameter is deprecated, don't show it\n                    continue\n\n            if not (isinstance(value, Sized) and len(value) == 0):\n                value = json.dumps(value)\n                parameters[key] = value\n            else:\n                parameters[key] = None\n\n        ret[\"oml-python:serialized_object\"] = \"cv_object\"\n        name = o.__module__ + \".\" + o.__class__.__name__\n        value = OrderedDict([(\"name\", name), (\"parameters\", parameters)])\n        ret[\"value\"] = value\n\n        return ret\n\n    def _deserialize_cross_validator(\n        self,\n        value: OrderedDict[str, Any],\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        model_name = value[\"name\"]\n        parameters = value[\"parameters\"]\n\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        for parameter in parameters:\n            parameters[parameter] = self._deserialize_sklearn(\n                parameters[parameter],\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n        return model_class(**parameters)\n\n    def _format_external_version(\n        self,\n        model_package_name: str,\n        model_package_version_number: str,\n    ) -&gt; str:\n        return f\"{model_package_name}=={model_package_version_number}\"\n\n    @staticmethod\n    def _get_parameter_values_recursive(\n        param_grid: dict | list[dict],\n        parameter_name: str,\n    ) -&gt; list[Any]:\n        \"\"\"\n        Returns a list of values for a given hyperparameter, encountered\n        recursively throughout the flow. (e.g., n_jobs can be defined\n        for various flows)\n\n        Parameters\n        ----------\n        param_grid: Union[Dict, List[Dict]]\n            Dict mapping from hyperparameter list to value, to a list of\n            such dicts\n\n        parameter_name: str\n            The hyperparameter that needs to be inspected\n\n        Returns\n        -------\n        List\n            A list of all values of hyperparameters with this name\n        \"\"\"\n        if isinstance(param_grid, dict):\n            return [\n                value\n                for param, value in param_grid.items()\n                if param.split(\"__\")[-1] == parameter_name\n            ]\n\n        if isinstance(param_grid, list):\n            result = []\n            for sub_grid in param_grid:\n                result.extend(\n                    SklearnExtension._get_parameter_values_recursive(sub_grid, parameter_name),\n                )\n            return result\n\n        raise ValueError(\"Param_grid should either be a dict or list of dicts\")\n\n    def _prevent_optimize_n_jobs(self, model):\n        \"\"\"\n        Ensures that HPO classes will not optimize the n_jobs hyperparameter\n\n        Parameters\n        ----------\n        model:\n            The model that will be fitted\n        \"\"\"\n        if self._is_hpo_class(model):\n            if isinstance(model, sklearn.model_selection.GridSearchCV):\n                param_distributions = model.param_grid\n            elif isinstance(model, sklearn.model_selection.RandomizedSearchCV):\n                param_distributions = model.param_distributions\n            else:\n                if hasattr(model, \"param_distributions\"):\n                    param_distributions = model.param_distributions\n                else:\n                    raise AttributeError(\n                        \"Using subclass BaseSearchCV other than \"\n                        \"{GridSearchCV, RandomizedSearchCV}. \"\n                        \"Could not find attribute \"\n                        \"param_distributions.\",\n                    )\n                logger.warning(\n                    \"Warning! Using subclass BaseSearchCV other than \"\n                    \"{GridSearchCV, RandomizedSearchCV}. \"\n                    \"Should implement param check. \",\n                )\n            n_jobs_vals = SklearnExtension._get_parameter_values_recursive(\n                param_distributions,\n                \"n_jobs\",\n            )\n            if len(n_jobs_vals) &gt; 0:\n                raise PyOpenMLError(\n                    \"openml-python should not be used to \" \"optimize the n_jobs parameter.\",\n                )\n\n    ################################################################################################\n    # Methods for performing runs with extension modules\n\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is a scikit-learn estimator.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        o = model\n        return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n\n    def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n        \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n        model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Models that are already seeded will maintain the seed. In this case,\n        only integer seeds are allowed (An exception is raised when a RandomState was used as\n        seed).\n\n        Parameters\n        ----------\n        model : sklearn model\n            The model to be seeded\n        seed : int\n            The seed to initialize the RandomState with. Unseeded subcomponents\n            will be seeded with a random number from the RandomState.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        def _seed_current_object(current_value):\n            if isinstance(current_value, int):  # acceptable behaviour\n                return False\n\n            if isinstance(current_value, np.random.RandomState):\n                raise ValueError(\n                    \"Models initialized with a RandomState object are not \"\n                    \"supported. Please seed with an integer. \",\n                )\n\n            if current_value is not None:\n                raise ValueError(\n                    \"Models should be seeded with int or None (this should never \" \"happen). \",\n                )\n\n            return True\n\n        rs = np.random.RandomState(seed)\n        model_params = model.get_params()\n        random_states = {}\n        for param_name in sorted(model_params):\n            if \"random_state\" in param_name:\n                current_value = model_params[param_name]\n                # important to draw the value at this point (and not in the if\n                # statement) this way we guarantee that if a different set of\n                # subflows is seeded, the same number of the random generator is\n                # used\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    random_states[param_name] = new_value\n\n            # Also seed CV objects!\n            elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n                if not hasattr(model_params[param_name], \"random_state\"):\n                    continue\n\n                current_value = model_params[param_name].random_state\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    model_params[param_name].random_state = new_value\n\n        model.set_params(**random_states)\n        return model\n\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        from sklearn.exceptions import NotFittedError\n        from sklearn.utils.validation import check_is_fitted\n\n        try:\n            # check if model is fitted\n            check_is_fitted(model)\n\n            # Creating random dummy data of arbitrary size\n            dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n            # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n            # robust check that works across sklearn versions and models. Internally, 'predict'\n            # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n            # assured check than explicit calls to 'check_is_fitted'\n            model.predict(dummy_data)\n            # Will reach here if the model was fit on a dataset with 3 features\n            return True\n        except NotFittedError:  # needs to be the first exception to be caught\n            # Model is not fitted, as is required\n            return False\n        except ValueError:\n            # Will reach here if the model was fit on a dataset with more or less than 3 features\n            return True\n\n    def _run_model_on_fold(  # noqa: PLR0915, PLR0913, C901, PLR0912\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame | None = None,\n    ) -&gt; tuple[\n        np.ndarray,\n        pd.DataFrame | None,\n        OrderedDict[str, float],\n        OpenMLRunTrace | None,\n    ]:\n        \"\"\"Run a model on a repeat,fold,subsample triplet of the task and return prediction\n        information.\n\n        Furthermore, it will measure run time measures in case multi-core behaviour allows this.\n        * exact user cpu time will be measured if the number of cores is set (recursive throughout\n        the model) exactly to 1\n        * wall clock time will be measured if the number of cores is set (recursive throughout the\n        model) to any given number (but not when it is set to -1)\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        run_task_get_arff_content. Do not use this function unless you know what you are doing.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        pred_y : np.ndarray\n            Predictions on the training/test set, depending on the task type.\n            For supervised tasks, predictions are on the test set.\n            For unsupervised tasks, predictions are on the training set.\n        proba_y : pd.DataFrame, optional\n            Predicted probabilities for the test set.\n            None, if task is not Classification or Learning Curve prediction.\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : OpenMLRunTrace, optional\n            arff trace object from a fitted model and the trace content obtained by\n            repeatedly calling ``run_model_on_task``\n        \"\"\"\n\n        def _prediction_to_probabilities(\n            y: np.ndarray | list,\n            model_classes: list[Any],\n            class_labels: list[str] | None,\n        ) -&gt; pd.DataFrame:\n            \"\"\"Transforms predicted probabilities to match with OpenML class indices.\n\n            Parameters\n            ----------\n            y : np.ndarray\n                Predicted probabilities (possibly omitting classes if they were not present in the\n                training data).\n            model_classes : list\n                List of classes known_predicted by the model, ordered by their index.\n            class_labels : list\n                List of classes as stored in the task object fetched from server.\n\n            Returns\n            -------\n            pd.DataFrame\n            \"\"\"\n            if class_labels is None:\n                raise ValueError(\"The task has no class labels\")\n\n            if isinstance(y_train, np.ndarray) and isinstance(class_labels[0], str):\n                # mapping (decoding) the predictions to the categories\n                # creating a separate copy to not change the expected pred_y type\n                y = [class_labels[pred] for pred in y]  # list or numpy array of predictions\n\n            # model_classes: sklearn classifier mapping from original array id to\n            # prediction index id\n            if not isinstance(model_classes, list):\n                raise ValueError(\"please convert model classes to list prior to calling this fn\")\n\n            # DataFrame allows more accurate mapping of classes as column names\n            result = pd.DataFrame(\n                0,\n                index=np.arange(len(y)),\n                columns=model_classes,\n                dtype=np.float32,\n            )\n            for obs, prediction in enumerate(y):\n                result.loc[obs, prediction] = 1.0\n            return result\n\n        if isinstance(task, OpenMLSupervisedTask):\n            if y_train is None:\n                raise TypeError(\"argument y_train must not be of type None\")\n            if X_test is None:\n                raise TypeError(\"argument X_test must not be of type None\")\n\n        model_copy = sklearn.base.clone(model, safe=True)\n        # sanity check: prohibit users from optimizing n_jobs\n        self._prevent_optimize_n_jobs(model_copy)\n        # measures and stores runtimes\n        user_defined_measures = OrderedDict()  # type: 'OrderedDict[str, float]'\n        try:\n            # for measuring runtime. Only available since Python 3.3\n            modelfit_start_cputime = time.process_time()\n            modelfit_start_walltime = time.time()\n\n            if isinstance(task, OpenMLSupervisedTask):\n                model_copy.fit(X_train, y_train)  # type: ignore\n            elif isinstance(task, OpenMLClusteringTask):\n                model_copy.fit(X_train)  # type: ignore\n\n            modelfit_dur_cputime = (time.process_time() - modelfit_start_cputime) * 1000\n            modelfit_dur_walltime = (time.time() - modelfit_start_walltime) * 1000\n\n            user_defined_measures[\"usercpu_time_millis_training\"] = modelfit_dur_cputime\n            refit_time = model_copy.refit_time_ * 1000 if hasattr(model_copy, \"refit_time_\") else 0  # type: ignore\n            user_defined_measures[\"wall_clock_time_millis_training\"] = modelfit_dur_walltime\n\n        except AttributeError as e:\n            # typically happens when training a regressor on classification task\n            raise PyOpenMLError(str(e)) from e\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            # search for model classes_ (might differ depending on modeltype)\n            # first, pipelines are a special case (these don't have a classes_\n            # object, but rather borrows it from the last step. We do this manually,\n            # because of the BaseSearch check)\n            if isinstance(model_copy, sklearn.pipeline.Pipeline):\n                used_estimator = model_copy.steps[-1][-1]\n            else:\n                used_estimator = model_copy\n\n            if self._is_hpo_class(used_estimator):\n                model_classes = used_estimator.best_estimator_.classes_\n            else:\n                model_classes = used_estimator.classes_\n\n            if not isinstance(model_classes, list):\n                model_classes = model_classes.tolist()\n\n            # to handle the case when dataset is numpy and categories are encoded\n            # however the class labels stored in task are still categories\n            if isinstance(y_train, np.ndarray) and isinstance(\n                cast(List, task.class_labels)[0],\n                str,\n            ):\n                model_classes = [cast(List[str], task.class_labels)[i] for i in model_classes]\n\n        modelpredict_start_cputime = time.process_time()\n        modelpredict_start_walltime = time.time()\n\n        # In supervised learning this returns the predictions for Y, in clustering\n        # it returns the clusters\n        if isinstance(task, OpenMLSupervisedTask):\n            pred_y = model_copy.predict(X_test)\n        elif isinstance(task, OpenMLClusteringTask):\n            pred_y = model_copy.predict(X_train)\n        else:\n            raise ValueError(task)\n\n        modelpredict_duration_cputime = (time.process_time() - modelpredict_start_cputime) * 1000\n        user_defined_measures[\"usercpu_time_millis_testing\"] = modelpredict_duration_cputime\n        user_defined_measures[\"usercpu_time_millis\"] = (\n            modelfit_dur_cputime + modelpredict_duration_cputime\n        )\n        modelpredict_duration_walltime = (time.time() - modelpredict_start_walltime) * 1000\n        user_defined_measures[\"wall_clock_time_millis_testing\"] = modelpredict_duration_walltime\n        user_defined_measures[\"wall_clock_time_millis\"] = (\n            modelfit_dur_walltime + modelpredict_duration_walltime + refit_time\n        )\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            try:\n                proba_y = model_copy.predict_proba(X_test)\n                proba_y = pd.DataFrame(proba_y, columns=model_classes)  # handles X_test as numpy\n            except AttributeError:  # predict_proba is not available when probability=False\n                proba_y = _prediction_to_probabilities(pred_y, model_classes, task.class_labels)\n\n            if task.class_labels is not None:\n                if proba_y.shape[1] != len(task.class_labels):\n                    # Remap the probabilities in case there was a class missing\n                    # at training time. By default, the classification targets\n                    # are mapped to be zero-based indices to the actual classes.\n                    # Therefore, the model_classes contain the correct indices to\n                    # the correct probability array. Example:\n                    # classes in the dataset: 0, 1, 2, 3, 4, 5\n                    # classes in the training set: 0, 1, 2, 4, 5\n                    # then we need to add a column full of zeros into the probabilities\n                    # for class 3 because the rest of the library expects that the\n                    # probabilities are ordered the same way as the classes are ordered).\n                    message = \"Estimator only predicted for {}/{} classes!\".format(\n                        proba_y.shape[1],\n                        len(task.class_labels),\n                    )\n                    warnings.warn(message, stacklevel=2)\n                    openml.config.logger.warning(message)\n\n                    for _i, col in enumerate(task.class_labels):\n                        # adding missing columns with 0 probability\n                        if col not in model_classes:\n                            proba_y[col] = 0\n                    # We re-order the columns to move possibly added missing columns into place.\n                    proba_y = proba_y[task.class_labels]\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n            if not np.all(set(proba_y.columns) == set(task.class_labels)):\n                missing_cols = list(set(task.class_labels) - set(proba_y.columns))\n                raise ValueError(\"Predicted probabilities missing for the columns: \", missing_cols)\n\n        elif isinstance(task, (OpenMLRegressionTask, OpenMLClusteringTask)):\n            proba_y = None\n        else:\n            raise TypeError(type(task))\n\n        if self._is_hpo_class(model_copy):\n            trace_data = self._extract_trace_data(model_copy, rep_no, fold_no)\n            trace: OpenMLRunTrace | None = self._obtain_arff_trace(\n                model_copy,\n                trace_data,\n            )\n        else:\n            trace = None\n\n        return pred_y, proba_y, user_defined_measures, trace\n\n    def obtain_parameter_values(  # noqa: C901, PLR0915\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n        openml.flows.functions._check_flow_for_server_id(flow)\n\n        def get_flow_dict(_flow):\n            flow_map = {_flow.name: _flow.flow_id}\n            for subflow in _flow.components:\n                flow_map.update(get_flow_dict(_flow.components[subflow]))\n            return flow_map\n\n        def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n            _flow,\n            _flow_dict,\n            component_model,\n            _main_call=False,  # noqa: FBT002\n            main_id=None,\n        ):\n            def is_subcomponent_specification(values):\n                # checks whether the current value can be a specification of\n                # subcomponents, as for example the value for steps parameter\n                # (in Pipeline) or transformers parameter (in\n                # ColumnTransformer).\n                return (\n                    # Specification requires list/tuple of list/tuple with\n                    # at least length 2.\n                    isinstance(values, (tuple, list))\n                    and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                    # And each component needs to be a flow or interpretable string\n                    and all(\n                        isinstance(item[1], openml.flows.OpenMLFlow)\n                        or (\n                            isinstance(item[1], str)\n                            and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        )\n                        for item in values\n                    )\n                )\n\n            # _flow is openml flow object, _param dict maps from flow name to flow\n            # id for the main call, the param dict can be overridden (useful for\n            # unit tests / sentinels) this way, for flows without subflows we do\n            # not have to rely on _flow_dict\n            exp_parameters = set(_flow.parameters)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_parameters = set()\n            else:\n                model_parameters = set(component_model.get_params(deep=False))\n            if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n                flow_params = sorted(exp_parameters)\n                model_params = sorted(model_parameters)\n                raise ValueError(\n                    \"Parameters of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow parameters: \"\n                    f\"{flow_params}\\nmodel parameters: {model_params}\",\n                )\n            exp_components = set(_flow.components)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_components = set()\n            else:\n                _ = set(component_model.get_params(deep=False))\n                model_components = {\n                    mp\n                    for mp in component_model.get_params(deep=True)\n                    if \"__\" not in mp and mp not in _\n                }\n            if len(exp_components.symmetric_difference(model_components)) != 0:\n                is_problem = True\n                if len(exp_components - model_components) &gt; 0:\n                    # If an expected component is not returned as a component by get_params(),\n                    # this means that it is also a parameter -&gt; we need to check that this is\n                    # actually the case\n                    difference = exp_components - model_components\n                    component_in_model_parameters = []\n                    for component in difference:\n                        if component in model_parameters:\n                            component_in_model_parameters.append(True)\n                        else:\n                            component_in_model_parameters.append(False)\n                    is_problem = not all(component_in_model_parameters)\n                if is_problem:\n                    flow_components = sorted(exp_components)\n                    model_components = sorted(model_components)\n                    raise ValueError(\n                        \"Subcomponents of the model do not match the \"\n                        \"parameters expected by the \"\n                        \"flow:\\nexpected flow subcomponents: \"\n                        f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                    )\n\n            _params = []\n            for _param_name in _flow.parameters:\n                _current = OrderedDict()\n                _current[\"oml:name\"] = _param_name\n\n                current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n                # Try to filter out components (a.k.a. subflows) which are\n                # handled further down in the code (by recursively calling\n                # this function)!\n                if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                    continue\n\n                if is_subcomponent_specification(current_param_values):\n                    # complex parameter value, with subcomponents\n                    parsed_values = []\n                    for subcomponent in current_param_values:\n                        # scikit-learn stores usually tuples in the form\n                        # (name (str), subcomponent (mixed), argument\n                        # (mixed)). OpenML replaces the subcomponent by an\n                        # OpenMLFlow object.\n                        if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                            raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                        subcomponent_identifier = subcomponent[0]\n                        subcomponent_flow = subcomponent[1]\n                        if not isinstance(subcomponent_identifier, str):\n                            raise TypeError(\n                                \"Subcomponent identifier should be of type string, \"\n                                f\"but is {type(subcomponent_identifier)}\",\n                            )\n                        if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                            if (\n                                isinstance(subcomponent_flow, str)\n                                and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                            ):\n                                pass\n                            else:\n                                raise TypeError(\n                                    \"Subcomponent flow should be of type flow, but is {}\".format(\n                                        type(subcomponent_flow),\n                                    ),\n                                )\n\n                        current = {\n                            \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                            \"value\": {\n                                \"key\": subcomponent_identifier,\n                                \"step_name\": subcomponent_identifier,\n                            },\n                        }\n                        if len(subcomponent) == 3:\n                            if not isinstance(subcomponent[2], list) and not isinstance(\n                                subcomponent[2],\n                                OrderedDict,\n                            ):\n                                raise TypeError(\n                                    \"Subcomponent argument should be list or OrderedDict\",\n                                )\n                            current[\"value\"][\"argument_1\"] = subcomponent[2]\n                        parsed_values.append(current)\n                    parsed_values = json.dumps(parsed_values)\n                else:\n                    # vanilla parameter value\n                    parsed_values = json.dumps(current_param_values)\n\n                _current[\"oml:value\"] = parsed_values\n                if _main_call:\n                    _current[\"oml:component\"] = main_id\n                else:\n                    _current[\"oml:component\"] = _flow_dict[_flow.name]\n                _params.append(_current)\n\n            for _identifier in _flow.components:\n                subcomponent_model = component_model.get_params()[_identifier]\n                _params.extend(\n                    extract_parameters(\n                        _flow.components[_identifier],\n                        _flow_dict,\n                        subcomponent_model,\n                    ),\n                )\n            return _params\n\n        flow_dict = get_flow_dict(flow)\n        model = model if model is not None else flow.model\n        return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n\n    def _openml_param_name_to_sklearn(\n        self,\n        openml_parameter: openml.setups.OpenMLParameter,\n        flow: OpenMLFlow,\n    ) -&gt; str:\n        \"\"\"\n        Converts the name of an OpenMLParameter into the sklean name, given a flow.\n\n        Parameters\n        ----------\n        openml_parameter: OpenMLParameter\n            The parameter under consideration\n\n        flow: OpenMLFlow\n            The flow that provides context.\n\n        Returns\n        -------\n        sklearn_parameter_name: str\n            The name the parameter will have once used in scikit-learn\n        \"\"\"\n        if not isinstance(openml_parameter, openml.setups.OpenMLParameter):\n            raise ValueError(\"openml_parameter should be an instance of OpenMLParameter\")\n        if not isinstance(flow, OpenMLFlow):\n            raise ValueError(\"flow should be an instance of OpenMLFlow\")\n\n        flow_structure = flow.get_structure(\"name\")\n        if openml_parameter.flow_name not in flow_structure:\n            raise ValueError(\"Obtained OpenMLParameter and OpenMLFlow do not correspond. \")\n        name = openml_parameter.flow_name  # for PEP8\n        return \"__\".join(flow_structure[name] + [openml_parameter.parameter_name])\n\n    ################################################################################################\n    # Methods for hyperparameter optimization\n\n    def _is_hpo_class(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the model performs hyperparameter optimization.\n\n        Used to check whether an optimization trace can be extracted from the model after\n        running it.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.model_selection._search.BaseSearchCV)\n\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n        optimization model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        base_estimator = model.estimator\n        base_estimator.set_params(**trace_iteration.get_parameters())\n        return base_estimator\n\n    def _extract_trace_data(self, model, rep_no, fold_no):\n        \"\"\"Extracts data from a machine learning model's cross-validation results\n        and creates an ARFF (Attribute-Relation File Format) trace.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n        rep_no : int\n            The repetition number.\n        fold_no : int\n            The fold number.\n\n        Returns\n        -------\n        A list of ARFF tracecontent.\n        \"\"\"\n        arff_tracecontent = []\n        for itt_no in range(len(model.cv_results_[\"mean_test_score\"])):\n            # we use the string values for True and False, as it is defined in\n            # this way by the OpenML server\n            selected = \"false\"\n            if itt_no == model.best_index_:\n                selected = \"true\"\n            test_score = model.cv_results_[\"mean_test_score\"][itt_no]\n            arff_line = [rep_no, fold_no, itt_no, test_score, selected]\n            for key in model.cv_results_:\n                if key.startswith(\"param_\"):\n                    value = model.cv_results_[key][itt_no]\n                    # Built-in serializer does not convert all numpy types,\n                    # these methods convert them to built-in types instead.\n                    if isinstance(value, np.generic):\n                        # For scalars it actually returns scalars, not a list\n                        value = value.tolist()\n                    serialized_value = json.dumps(value) if value is not np.ma.masked else np.nan\n                    arff_line.append(serialized_value)\n            arff_tracecontent.append(arff_line)\n        return arff_tracecontent\n\n    def _obtain_arff_trace(\n        self,\n        model: Any,\n        trace_content: list,\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Create arff trace object from a fitted model and the trace content obtained by\n        repeatedly calling ``run_model_on_task``.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n\n        trace_content : List[List]\n            Trace content obtained by ``openml.runs.run_flow_on_task``.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        if not hasattr(model, \"cv_results_\"):\n            raise ValueError(\"model should contain `cv_results_`\")\n\n        # attributes that will be in trace arff, regardless of the model\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n\n        # model dependent attributes for trace arff\n        for key in model.cv_results_:\n            if key.startswith(\"param_\"):\n                # supported types should include all types, including bool,\n                # int float\n                supported_basic_types = (bool, int, float, str)\n                for param_value in model.cv_results_[key]:\n                    if isinstance(param_value, np.generic):\n                        param_value = param_value.tolist()  # noqa: PLW2901\n                    if (\n                        isinstance(param_value, supported_basic_types)\n                        or param_value is None\n                        or param_value is np.ma.masked\n                    ):\n                        # basic string values\n                        type = \"STRING\"  # noqa: A001\n                    elif isinstance(param_value, (list, tuple)) and all(\n                        isinstance(i, int) for i in param_value\n                    ):\n                        # list of integers (usually for selecting features)\n                        # hyperparameter layer_sizes of MLPClassifier\n                        type = \"STRING\"  # noqa: A001\n                    else:\n                        raise TypeError(\"Unsupported param type in param grid: %s\" % key)\n\n                # renamed the attribute param to parameter, as this is a required\n                # OpenML convention - this also guards against name collisions\n                # with the required trace attributes\n                attribute = (PREFIX + key[6:], type)  # type: ignore\n                trace_attributes.append(attribute)\n\n        return OpenMLRunTrace.generate(\n            trace_attributes,\n            trace_content,\n        )\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>classmethod</code>","text":"<p>Check whether a given describes a scikit-learn estimator.</p> <p>This is done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given describes a scikit-learn estimator.\n\n    This is done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return cls._is_sklearn_flow(flow)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>classmethod</code>","text":"<p>Check whether a model is an instance of <code>sklearn.base.BaseEstimator</code>.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return isinstance(model, sklearn.base.BaseEstimator)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    from sklearn.exceptions import NotFittedError\n    from sklearn.utils.validation import check_is_fitted\n\n    try:\n        # check if model is fitted\n        check_is_fitted(model)\n\n        # Creating random dummy data of arbitrary size\n        dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n        # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n        # robust check that works across sklearn versions and models. Internally, 'predict'\n        # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n        # assured check than explicit calls to 'check_is_fitted'\n        model.predict(dummy_data)\n        # Will reach here if the model was fit on a dataset with 3 features\n        return True\n    except NotFittedError:  # needs to be the first exception to be caught\n        # Model is not fitted, as is required\n        return False\n    except ValueError:\n        # Will reach here if the model was fit on a dataset with more or less than 3 features\n        return True\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.create_setup_string","title":"<code>create_setup_string(model)</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n    return \" \".join(self.get_version_information())\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>","text":"<p>Initializes a sklearn model based on a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>mixed</code> <p>the object to deserialize (can be flow object, or any serialized parameter value that is accepted by)</p> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>mixed</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; Any:\n    \"\"\"Initializes a sklearn model based on a flow.\n\n    Parameters\n    ----------\n    flow : mixed\n        the object to deserialize (can be flow object, or any serialized\n        parameter value that is accepted by)\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    mixed\n    \"\"\"\n    return self._deserialize_sklearn(\n        flow,\n        initialize_with_defaults=initialize_with_defaults,\n        strict_version=strict_version,\n    )\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.get_version_information","title":"<code>get_version_information()</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Libraries listed are <code>Python</code>, <code>scikit-learn</code>, <code>numpy</code> and <code>scipy</code>.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n    Returns\n    -------\n    List\n    \"\"\"\n    # This can possibly be done by a package such as pyxb, but I could not get\n    # it to work properly.\n    import numpy\n    import scipy\n    import sklearn\n\n    major, minor, micro, _, _ = sys.version_info\n    python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n    sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n    numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n    scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n    return [python_version, sklearn_version, numpy_version, scipy_version]\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>","text":"<p>Instantiate a <code>base_estimator</code> which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n    optimization model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n    if not self._is_hpo_class(model):\n        raise AssertionError(\n            \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n            % model,\n        )\n    base_estimator = model.estimator\n    base_estimator.set_params(**trace_iteration.get_parameters())\n    return base_estimator\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.is_estimator","title":"<code>is_estimator(model)</code>","text":"<p>Check whether the given model is a scikit-learn estimator.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is a scikit-learn estimator.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    o = model\n    return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.model_to_flow","title":"<code>model_to_flow(model)</code>","text":"<p>Transform a scikit-learn model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n    # Necessary to make pypy not complain about all the different possible return types\n    return self._serialize_sklearn(model)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def obtain_parameter_values(  # noqa: C901, PLR0915\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n    openml.flows.functions._check_flow_for_server_id(flow)\n\n    def get_flow_dict(_flow):\n        flow_map = {_flow.name: _flow.flow_id}\n        for subflow in _flow.components:\n            flow_map.update(get_flow_dict(_flow.components[subflow]))\n        return flow_map\n\n    def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n        _flow,\n        _flow_dict,\n        component_model,\n        _main_call=False,  # noqa: FBT002\n        main_id=None,\n    ):\n        def is_subcomponent_specification(values):\n            # checks whether the current value can be a specification of\n            # subcomponents, as for example the value for steps parameter\n            # (in Pipeline) or transformers parameter (in\n            # ColumnTransformer).\n            return (\n                # Specification requires list/tuple of list/tuple with\n                # at least length 2.\n                isinstance(values, (tuple, list))\n                and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                # And each component needs to be a flow or interpretable string\n                and all(\n                    isinstance(item[1], openml.flows.OpenMLFlow)\n                    or (\n                        isinstance(item[1], str)\n                        and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    )\n                    for item in values\n                )\n            )\n\n        # _flow is openml flow object, _param dict maps from flow name to flow\n        # id for the main call, the param dict can be overridden (useful for\n        # unit tests / sentinels) this way, for flows without subflows we do\n        # not have to rely on _flow_dict\n        exp_parameters = set(_flow.parameters)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_parameters = set()\n        else:\n            model_parameters = set(component_model.get_params(deep=False))\n        if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n            flow_params = sorted(exp_parameters)\n            model_params = sorted(model_parameters)\n            raise ValueError(\n                \"Parameters of the model do not match the \"\n                \"parameters expected by the \"\n                \"flow:\\nexpected flow parameters: \"\n                f\"{flow_params}\\nmodel parameters: {model_params}\",\n            )\n        exp_components = set(_flow.components)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_components = set()\n        else:\n            _ = set(component_model.get_params(deep=False))\n            model_components = {\n                mp\n                for mp in component_model.get_params(deep=True)\n                if \"__\" not in mp and mp not in _\n            }\n        if len(exp_components.symmetric_difference(model_components)) != 0:\n            is_problem = True\n            if len(exp_components - model_components) &gt; 0:\n                # If an expected component is not returned as a component by get_params(),\n                # this means that it is also a parameter -&gt; we need to check that this is\n                # actually the case\n                difference = exp_components - model_components\n                component_in_model_parameters = []\n                for component in difference:\n                    if component in model_parameters:\n                        component_in_model_parameters.append(True)\n                    else:\n                        component_in_model_parameters.append(False)\n                is_problem = not all(component_in_model_parameters)\n            if is_problem:\n                flow_components = sorted(exp_components)\n                model_components = sorted(model_components)\n                raise ValueError(\n                    \"Subcomponents of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow subcomponents: \"\n                    f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                )\n\n        _params = []\n        for _param_name in _flow.parameters:\n            _current = OrderedDict()\n            _current[\"oml:name\"] = _param_name\n\n            current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n            # Try to filter out components (a.k.a. subflows) which are\n            # handled further down in the code (by recursively calling\n            # this function)!\n            if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                continue\n\n            if is_subcomponent_specification(current_param_values):\n                # complex parameter value, with subcomponents\n                parsed_values = []\n                for subcomponent in current_param_values:\n                    # scikit-learn stores usually tuples in the form\n                    # (name (str), subcomponent (mixed), argument\n                    # (mixed)). OpenML replaces the subcomponent by an\n                    # OpenMLFlow object.\n                    if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                        raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                    subcomponent_identifier = subcomponent[0]\n                    subcomponent_flow = subcomponent[1]\n                    if not isinstance(subcomponent_identifier, str):\n                        raise TypeError(\n                            \"Subcomponent identifier should be of type string, \"\n                            f\"but is {type(subcomponent_identifier)}\",\n                        )\n                    if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                        if (\n                            isinstance(subcomponent_flow, str)\n                            and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        ):\n                            pass\n                        else:\n                            raise TypeError(\n                                \"Subcomponent flow should be of type flow, but is {}\".format(\n                                    type(subcomponent_flow),\n                                ),\n                            )\n\n                    current = {\n                        \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                        \"value\": {\n                            \"key\": subcomponent_identifier,\n                            \"step_name\": subcomponent_identifier,\n                        },\n                    }\n                    if len(subcomponent) == 3:\n                        if not isinstance(subcomponent[2], list) and not isinstance(\n                            subcomponent[2],\n                            OrderedDict,\n                        ):\n                            raise TypeError(\n                                \"Subcomponent argument should be list or OrderedDict\",\n                            )\n                        current[\"value\"][\"argument_1\"] = subcomponent[2]\n                    parsed_values.append(current)\n                parsed_values = json.dumps(parsed_values)\n            else:\n                # vanilla parameter value\n                parsed_values = json.dumps(current_param_values)\n\n            _current[\"oml:value\"] = parsed_values\n            if _main_call:\n                _current[\"oml:component\"] = main_id\n            else:\n                _current[\"oml:component\"] = _flow_dict[_flow.name]\n            _params.append(_current)\n\n        for _identifier in _flow.components:\n            subcomponent_model = component_model.get_params()[_identifier]\n            _params.extend(\n                extract_parameters(\n                    _flow.components[_identifier],\n                    _flow_dict,\n                    subcomponent_model,\n                ),\n            )\n        return _params\n\n    flow_dict = get_flow_dict(flow)\n    model = model if model is not None else flow.model\n    return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.seed_model","title":"<code>seed_model(model, seed=None)</code>","text":"<p>Set the random state of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is raised when a RandomState was used as seed).</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> <p>The seed to initialize the RandomState with. Unseeded subcomponents will be seeded with a random number from the RandomState.</p> <code>None</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n    \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n    model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Models that are already seeded will maintain the seed. In this case,\n    only integer seeds are allowed (An exception is raised when a RandomState was used as\n    seed).\n\n    Parameters\n    ----------\n    model : sklearn model\n        The model to be seeded\n    seed : int\n        The seed to initialize the RandomState with. Unseeded subcomponents\n        will be seeded with a random number from the RandomState.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    def _seed_current_object(current_value):\n        if isinstance(current_value, int):  # acceptable behaviour\n            return False\n\n        if isinstance(current_value, np.random.RandomState):\n            raise ValueError(\n                \"Models initialized with a RandomState object are not \"\n                \"supported. Please seed with an integer. \",\n            )\n\n        if current_value is not None:\n            raise ValueError(\n                \"Models should be seeded with int or None (this should never \" \"happen). \",\n            )\n\n        return True\n\n    rs = np.random.RandomState(seed)\n    model_params = model.get_params()\n    random_states = {}\n    for param_name in sorted(model_params):\n        if \"random_state\" in param_name:\n            current_value = model_params[param_name]\n            # important to draw the value at this point (and not in the if\n            # statement) this way we guarantee that if a different set of\n            # subflows is seeded, the same number of the random generator is\n            # used\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                random_states[param_name] = new_value\n\n        # Also seed CV objects!\n        elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n            if not hasattr(model_params[param_name], \"random_state\"):\n                continue\n\n            current_value = model_params[param_name].random_state\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                model_params[param_name].random_state = new_value\n\n    model.set_params(**random_states)\n    return model\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.trim_flow_name","title":"<code>trim_flow_name(long_name, extra_trim_length=100, _outer=True)</code>  <code>classmethod</code>","text":"<p>Shorten generated sklearn flow name to at most <code>max_length</code> characters.</p> <p>Flows are assumed to have the following naming structure: <code>(model_selection)? (pipeline)? (steps)+</code> and will be shortened to: <code>sklearn.(selection.)?(pipeline.)?(steps)+</code> e.g. (white spaces and newlines added for readability)</p> <p>.. code ::</p> <pre><code>sklearn.pipeline.Pipeline(\n    columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n        numeric=sklearn.pipeline.Pipeline(\n            imputer=sklearn.preprocessing.imputation.Imputer,\n            standardscaler=sklearn.preprocessing.data.StandardScaler),\n        nominal=sklearn.pipeline.Pipeline(\n            simpleimputer=sklearn.impute.SimpleImputer,\n            onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n    variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n    svc=sklearn.svm.classes.SVC)\n</code></pre> <p>-&gt; <code>sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)</code></p> <p>Parameters:</p> Name Type Description Default <code>long_name</code> <code>str</code> <p>The full flow name generated by the scikit-learn extension.</p> required <code>extra_trim_length</code> <code>int</code> <p>If the trimmed name would exceed <code>extra_trim_length</code> characters, additional trimming of the short name is performed. This reduces the produced short name length. There is no guarantee the end result will not exceed <code>extra_trim_length</code>.</p> <code>100</code> <code>_outer</code> <code>bool(default=True)</code> <p>For internal use only. Specifies if the function is called recursively.</p> <code>True</code> <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef trim_flow_name(  # noqa: C901\n    cls,\n    long_name: str,\n    extra_trim_length: int = 100,\n    _outer: bool = True,  # noqa: FBT001, FBT002\n) -&gt; str:\n    \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n    Flows are assumed to have the following naming structure:\n    ``(model_selection)? (pipeline)? (steps)+``\n    and will be shortened to:\n    ``sklearn.(selection.)?(pipeline.)?(steps)+``\n    e.g. (white spaces and newlines added for readability)\n\n    .. code ::\n\n        sklearn.pipeline.Pipeline(\n            columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                numeric=sklearn.pipeline.Pipeline(\n                    imputer=sklearn.preprocessing.imputation.Imputer,\n                    standardscaler=sklearn.preprocessing.data.StandardScaler),\n                nominal=sklearn.pipeline.Pipeline(\n                    simpleimputer=sklearn.impute.SimpleImputer,\n                    onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n            variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n            svc=sklearn.svm.classes.SVC)\n\n    -&gt;\n    ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n    Parameters\n    ----------\n    long_name : str\n        The full flow name generated by the scikit-learn extension.\n    extra_trim_length: int (default=100)\n        If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n        of the short name is performed. This reduces the produced short name length.\n        There is no guarantee the end result will not exceed `extra_trim_length`.\n    _outer : bool (default=True)\n        For internal use only. Specifies if the function is called recursively.\n\n    Returns\n    -------\n    str\n\n    \"\"\"\n\n    def remove_all_in_parentheses(string: str) -&gt; str:\n        string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        while removals &gt; 0:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        return string\n\n    # Generally, we want to trim all hyperparameters, the exception to that is for model\n    # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n    # So we first trim name of the `estimator` specified in mode selection. For reference, in\n    # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n    # keep it in the final trimmed flow name:\n    # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n    # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n    # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n    # sklearn.tree.tree.DecisionTreeClassifier))\n    if \"sklearn.model_selection\" in long_name:\n        start_index = long_name.index(\"sklearn.model_selection\")\n        estimator_start = (\n            start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n        )\n\n        model_select_boilerplate = long_name[start_index:estimator_start]\n        # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n        model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n        # Now we want to also find and parse the `estimator`, for this we find the closing\n        # parenthesis to the model selection technique:\n        closing_parenthesis_expected = 1\n        for char in long_name[estimator_start:]:\n            if char == \"(\":\n                closing_parenthesis_expected += 1\n            if char == \")\":\n                closing_parenthesis_expected -= 1\n            if closing_parenthesis_expected == 0:\n                break\n\n        _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n        model_select_pipeline = long_name[estimator_start:_end]\n\n        trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n        _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n        model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n        name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n    else:\n        name = long_name\n\n    module_name = long_name.split(\".\")[0]\n    short_name = module_name + \".{}\"\n\n    if name.startswith(\"sklearn.pipeline\"):\n        full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n        pipeline_class = full_pipeline_class.split(\".\")[-1]\n        # We don't want nested pipelines in the short name, so we trim all complicated\n        # subcomponents, i.e. those with parentheses:\n        pipeline = remove_all_in_parentheses(pipeline)\n\n        # then the pipeline steps are formatted e.g.:\n        # step1name=sklearn.submodule.ClassName,step2name...\n        components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n        pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n        if len(short_name.format(pipeline)) &gt; extra_trim_length:\n            pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n    else:\n        # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n        pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n    if not _outer:\n        # Anything from parenthesis in inner calls should not be culled, so we use brackets\n        pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n    else:\n        # Square brackets may be introduced with nested model_selection\n        pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n    return short_name.format(pipeline)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.cat","title":"<code>cat(X)</code>","text":"<p>Returns True for all categorical columns, False for the rest.</p> <p>This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is required to process each type of columns separately. This function allows transformations meant for categorical columns to access the categorical columns given the dataset as DataFrame.</p> Source code in <code>openml/extensions/sklearn/__init__.py</code> <pre><code>def cat(X: pd.DataFrame) -&gt; pd.Series:\n    \"\"\"Returns True for all categorical columns, False for the rest.\n\n    This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling\n    of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is\n    required to process each type of columns separately.\n    This function allows transformations meant for categorical columns to access the\n    categorical columns given the dataset as DataFrame.\n    \"\"\"\n    if not hasattr(X, \"dtypes\"):\n        raise AttributeError(\"Not a Pandas DataFrame with 'dtypes' as attribute!\")\n    return X.dtypes == \"category\"\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.cont","title":"<code>cont(X)</code>","text":"<p>Returns True for all non-categorical columns, False for the rest.</p> <p>This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is required to process each type of columns separately. This function allows transformations meant for continuous/numeric columns to access the continuous/numeric columns given the dataset as DataFrame.</p> Source code in <code>openml/extensions/sklearn/__init__.py</code> <pre><code>def cont(X: pd.DataFrame) -&gt; pd.Series:\n    \"\"\"Returns True for all non-categorical columns, False for the rest.\n\n    This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling\n    of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is\n    required to process each type of columns separately.\n    This function allows transformations meant for continuous/numeric columns to access the\n    continuous/numeric columns given the dataset as DataFrame.\n    \"\"\"\n    if not hasattr(X, \"dtypes\"):\n        raise AttributeError(\"Not a Pandas DataFrame with 'dtypes' as attribute!\")\n    return X.dtypes != \"category\"\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/","title":"extension","text":""},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension","title":"<code>SklearnExtension</code>","text":"<p>               Bases: <code>Extension</code></p> <p>Connect scikit-learn to OpenML-Python. The estimators which use this extension must be scikit-learn compatible, i.e needs to be a subclass of sklearn.base.BaseEstimator\".</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>class SklearnExtension(Extension):\n    \"\"\"Connect scikit-learn to OpenML-Python.\n    The estimators which use this extension must be scikit-learn compatible,\n    i.e needs to be a subclass of sklearn.base.BaseEstimator\".\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given describes a scikit-learn estimator.\n\n        This is done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return cls._is_sklearn_flow(flow)\n\n    @classmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.base.BaseEstimator)\n\n    @classmethod\n    def trim_flow_name(  # noqa: C901\n        cls,\n        long_name: str,\n        extra_trim_length: int = 100,\n        _outer: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; str:\n        \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n        Flows are assumed to have the following naming structure:\n        ``(model_selection)? (pipeline)? (steps)+``\n        and will be shortened to:\n        ``sklearn.(selection.)?(pipeline.)?(steps)+``\n        e.g. (white spaces and newlines added for readability)\n\n        .. code ::\n\n            sklearn.pipeline.Pipeline(\n                columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                    numeric=sklearn.pipeline.Pipeline(\n                        imputer=sklearn.preprocessing.imputation.Imputer,\n                        standardscaler=sklearn.preprocessing.data.StandardScaler),\n                    nominal=sklearn.pipeline.Pipeline(\n                        simpleimputer=sklearn.impute.SimpleImputer,\n                        onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n                variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n                svc=sklearn.svm.classes.SVC)\n\n        -&gt;\n        ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n        Parameters\n        ----------\n        long_name : str\n            The full flow name generated by the scikit-learn extension.\n        extra_trim_length: int (default=100)\n            If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n            of the short name is performed. This reduces the produced short name length.\n            There is no guarantee the end result will not exceed `extra_trim_length`.\n        _outer : bool (default=True)\n            For internal use only. Specifies if the function is called recursively.\n\n        Returns\n        -------\n        str\n\n        \"\"\"\n\n        def remove_all_in_parentheses(string: str) -&gt; str:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            while removals &gt; 0:\n                string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            return string\n\n        # Generally, we want to trim all hyperparameters, the exception to that is for model\n        # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n        # So we first trim name of the `estimator` specified in mode selection. For reference, in\n        # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n        # keep it in the final trimmed flow name:\n        # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n        # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n        # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n        # sklearn.tree.tree.DecisionTreeClassifier))\n        if \"sklearn.model_selection\" in long_name:\n            start_index = long_name.index(\"sklearn.model_selection\")\n            estimator_start = (\n                start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n            )\n\n            model_select_boilerplate = long_name[start_index:estimator_start]\n            # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n            model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n            # Now we want to also find and parse the `estimator`, for this we find the closing\n            # parenthesis to the model selection technique:\n            closing_parenthesis_expected = 1\n            for char in long_name[estimator_start:]:\n                if char == \"(\":\n                    closing_parenthesis_expected += 1\n                if char == \")\":\n                    closing_parenthesis_expected -= 1\n                if closing_parenthesis_expected == 0:\n                    break\n\n            _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n            model_select_pipeline = long_name[estimator_start:_end]\n\n            trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n            _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n            model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n            name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n        else:\n            name = long_name\n\n        module_name = long_name.split(\".\")[0]\n        short_name = module_name + \".{}\"\n\n        if name.startswith(\"sklearn.pipeline\"):\n            full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n            pipeline_class = full_pipeline_class.split(\".\")[-1]\n            # We don't want nested pipelines in the short name, so we trim all complicated\n            # subcomponents, i.e. those with parentheses:\n            pipeline = remove_all_in_parentheses(pipeline)\n\n            # then the pipeline steps are formatted e.g.:\n            # step1name=sklearn.submodule.ClassName,step2name...\n            components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n            pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n            if len(short_name.format(pipeline)) &gt; extra_trim_length:\n                pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n        else:\n            # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n            pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n        if not _outer:\n            # Anything from parenthesis in inner calls should not be culled, so we use brackets\n            pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n        else:\n            # Square brackets may be introduced with nested model_selection\n            pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n        return short_name.format(pipeline)\n\n    @classmethod\n    def _min_dependency_str(cls, sklearn_version: str) -&gt; str:\n        \"\"\"Returns a string containing the minimum dependencies for the sklearn version passed.\n\n        Parameters\n        ----------\n        sklearn_version : str\n            A version string of the xx.xx.xx\n\n        Returns\n        -------\n        str\n        \"\"\"\n        openml_major_version = int(LooseVersion(openml.__version__).version[1])\n        # This explicit check is necessary to support existing entities on the OpenML servers\n        # that used the fixed dependency string (in the else block)\n        if openml_major_version &gt; 11:\n            # OpenML v0.11 onwards supports sklearn&gt;=0.24\n            # assumption: 0.24 onwards sklearn should contain a _min_dependencies.py file with\n            # variables declared for extracting minimum dependency for that version\n            if LooseVersion(sklearn_version) &gt;= \"0.24\":\n                from sklearn import _min_dependencies as _mindep\n\n                dependency_list = {\n                    \"numpy\": f\"{_mindep.NUMPY_MIN_VERSION}\",\n                    \"scipy\": f\"{_mindep.SCIPY_MIN_VERSION}\",\n                    \"joblib\": f\"{_mindep.JOBLIB_MIN_VERSION}\",\n                    \"threadpoolctl\": f\"{_mindep.THREADPOOLCTL_MIN_VERSION}\",\n                }\n            elif LooseVersion(sklearn_version) &gt;= \"0.23\":\n                dependency_list = {\n                    \"numpy\": \"1.13.3\",\n                    \"scipy\": \"0.19.1\",\n                    \"joblib\": \"0.11\",\n                    \"threadpoolctl\": \"2.0.0\",\n                }\n                if LooseVersion(sklearn_version).version[2] == 0:\n                    dependency_list.pop(\"threadpoolctl\")\n            elif LooseVersion(sklearn_version) &gt;= \"0.21\":\n                dependency_list = {\"numpy\": \"1.11.0\", \"scipy\": \"0.17.0\", \"joblib\": \"0.11\"}\n            elif LooseVersion(sklearn_version) &gt;= \"0.19\":\n                dependency_list = {\"numpy\": \"1.8.2\", \"scipy\": \"0.13.3\"}\n            else:\n                dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n        else:\n            # this is INCORRECT for sklearn versions &gt;= 0.19 and &lt; 0.24\n            # given that OpenML has existing flows uploaded with such dependency information,\n            # we change no behaviour for older sklearn version, however from 0.24 onwards\n            # the dependency list will be accurately updated for any flow uploaded to OpenML\n            dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n\n        sklearn_dep = f\"sklearn=={sklearn_version}\"\n        dep_str = \"\\n\".join([f\"{k}&gt;={v}\" for k, v in dependency_list.items()])\n        return \"\\n\".join([sklearn_dep, dep_str])\n\n    ################################################################################################\n    # Methods for flow serialization and de-serialization\n\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; Any:\n        \"\"\"Initializes a sklearn model based on a flow.\n\n        Parameters\n        ----------\n        flow : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        return self._deserialize_sklearn(\n            flow,\n            initialize_with_defaults=initialize_with_defaults,\n            strict_version=strict_version,\n        )\n\n    def _deserialize_sklearn(  # noqa: PLR0915, C901, PLR0913, PLR0912\n        self,\n        o: Any,\n        components: dict | None = None,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        recursion_depth: int = 0,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Recursive function to deserialize a scikit-learn flow.\n\n        This function inspects an object to deserialize and decides how to do so. This function\n        delegates all work to the respective functions to deserialize special data structures etc.\n        This function works on everything that has been serialized to OpenML: OpenMLFlow,\n        components (which are flows themselves), functions, hyperparameter distributions (for\n        random search) and the actual hyperparameter values themselves.\n\n        Parameters\n        ----------\n        o : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        components : Optional[dict]\n            Components of the current flow being de-serialized. These will not be used when\n            de-serializing the actual flow, but when de-serializing a component reference.\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        recursion_depth : int\n            The depth at which this flow is called, mostly for debugging\n            purposes\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        logger.info(\n            \"-{} flow_to_sklearn START o={}, components={}, init_defaults={}\".format(\n                \"-\" * recursion_depth, o, components, initialize_with_defaults\n            ),\n        )\n        depth_pp = recursion_depth + 1  # shortcut var, depth plus plus\n\n        # First, we need to check whether the presented object is a json string.\n        # JSON strings are used to encoder parameter values. By passing around\n        # json strings for parameters, we make sure that we can flow_to_sklearn\n        # the parameter values to the correct type.\n\n        if isinstance(o, str):\n            with contextlib.suppress(JSONDecodeError):\n                o = json.loads(o)\n\n        if isinstance(o, dict):\n            # Check if the dict encodes a 'special' object, which could not\n            # easily converted into a string, but rather the information to\n            # re-create the object were stored in a dictionary.\n            if \"oml-python:serialized_object\" in o:\n                serialized_type = o[\"oml-python:serialized_object\"]\n                value = o[\"value\"]\n                if serialized_type == \"type\":\n                    rval = self._deserialize_type(value)\n                elif serialized_type == \"rv_frozen\":\n                    rval = self._deserialize_rv_frozen(value)\n                elif serialized_type == \"function\":\n                    rval = self._deserialize_function(value)\n                elif serialized_type in (COMPOSITION_STEP_CONSTANT, COMPONENT_REFERENCE):\n                    if serialized_type == COMPOSITION_STEP_CONSTANT:\n                        pass\n                    elif serialized_type == COMPONENT_REFERENCE:\n                        value = self._deserialize_sklearn(\n                            value,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        )\n                    else:\n                        raise NotImplementedError(serialized_type)\n                    assert components is not None  # Necessary for mypy\n                    step_name = value[\"step_name\"]\n                    key = value[\"key\"]\n                    component = self._deserialize_sklearn(\n                        components[key],\n                        initialize_with_defaults=initialize_with_defaults,\n                        recursion_depth=depth_pp,\n                        strict_version=strict_version,\n                    )\n                    # The component is now added to where it should be used\n                    # later. It should not be passed to the constructor of the\n                    # main flow object.\n                    del components[key]\n                    if step_name is None:\n                        rval = component\n                    elif \"argument_1\" not in value:\n                        rval = (step_name, component)\n                    else:\n                        rval = (step_name, component, value[\"argument_1\"])\n                elif serialized_type == \"cv_object\":\n                    rval = self._deserialize_cross_validator(\n                        value,\n                        recursion_depth=recursion_depth,\n                        strict_version=strict_version,\n                    )\n                else:\n                    raise ValueError(\"Cannot flow_to_sklearn %s\" % serialized_type)\n\n            else:\n                rval = OrderedDict(\n                    (\n                        self._deserialize_sklearn(\n                            o=key,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                        self._deserialize_sklearn(\n                            o=value,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                    )\n                    for key, value in sorted(o.items())\n                )\n        elif isinstance(o, (list, tuple)):\n            rval = [\n                self._deserialize_sklearn(\n                    o=element,\n                    components=components,\n                    initialize_with_defaults=initialize_with_defaults,\n                    recursion_depth=depth_pp,\n                    strict_version=strict_version,\n                )\n                for element in o\n            ]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, (bool, int, float, str)) or o is None:\n            rval = o\n        elif isinstance(o, OpenMLFlow):\n            if not self._is_sklearn_flow(o):\n                raise ValueError(\"Only sklearn flows can be reinstantiated\")\n            rval = self._deserialize_model(\n                flow=o,\n                keep_defaults=initialize_with_defaults,\n                recursion_depth=recursion_depth,\n                strict_version=strict_version,\n            )\n        else:\n            raise TypeError(o)\n        logger.info(\n            \"-{} flow_to_sklearn END   o={}, rval={}\".format(\"-\" * recursion_depth, o, rval)\n        )\n        return rval\n\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n        # Necessary to make pypy not complain about all the different possible return types\n        return self._serialize_sklearn(model)\n\n    def _serialize_sklearn(self, o: Any, parent_model: Any | None = None) -&gt; Any:  # noqa: PLR0912, C901\n        rval = None  # type: Any\n\n        # TODO: assert that only on first recursion lvl `parent_model` can be None\n        if self.is_estimator(o):\n            # is the main model or a submodel\n            rval = self._serialize_model(o)\n        elif (\n            isinstance(o, (list, tuple))\n            and len(o) == 2\n            and o[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            and isinstance(parent_model, sklearn.pipeline._BaseComposition)\n        ):\n            rval = o\n        elif isinstance(o, (list, tuple)):\n            # TODO: explain what type of parameter is here\n            rval = [self._serialize_sklearn(element, parent_model) for element in o]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, SIMPLE_TYPES) or o is None:\n            if isinstance(o, tuple(SIMPLE_NUMPY_TYPES)):\n                o = o.item()  # type: ignore\n            # base parameter values\n            rval = o\n        elif isinstance(o, dict):\n            # TODO: explain what type of parameter is here\n            if not isinstance(o, OrderedDict):\n                o = OrderedDict(sorted(o.items()))\n\n            rval = OrderedDict()\n            for key, value in o.items():\n                if not isinstance(key, str):\n                    raise TypeError(\n                        \"Can only use string as keys, you passed \"\n                        f\"type {type(key)} for value {key!s}.\",\n                    )\n                _key = self._serialize_sklearn(key, parent_model)\n                rval[_key] = self._serialize_sklearn(value, parent_model)\n        elif isinstance(o, type):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_type(o)\n        elif isinstance(o, scipy.stats.distributions.rv_frozen):\n            rval = self._serialize_rv_frozen(o)\n        # This only works for user-defined functions (and not even partial).\n        # I think this is exactly what we want here as there shouldn't be any\n        # built-in or functool.partials in a pipeline\n        elif inspect.isfunction(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_function(o)\n        elif self._is_cross_validator(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_cross_validator(o)\n        else:\n            raise TypeError(o, type(o))\n\n        return rval\n\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n        Returns\n        -------\n        List\n        \"\"\"\n        # This can possibly be done by a package such as pyxb, but I could not get\n        # it to work properly.\n        import numpy\n        import scipy\n        import sklearn\n\n        major, minor, micro, _, _ = sys.version_info\n        python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n        sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n        numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n        scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n        return [python_version, sklearn_version, numpy_version, scipy_version]\n\n    def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return \" \".join(self.get_version_information())\n\n    def _is_cross_validator(self, o: Any) -&gt; bool:\n        return isinstance(o, sklearn.model_selection.BaseCrossValidator)\n\n    @classmethod\n    def _is_sklearn_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        sklearn_dependency = isinstance(flow.dependencies, str) and \"sklearn\" in flow.dependencies\n        sklearn_as_external = isinstance(flow.external_version, str) and (\n            flow.external_version.startswith(\"sklearn==\") or \",sklearn==\" in flow.external_version\n        )\n        return sklearn_dependency or sklearn_as_external\n\n    def _get_sklearn_description(self, model: Any, char_lim: int = 1024) -&gt; str:\n        r\"\"\"Fetches the sklearn function docstring for the flow description\n\n        Retrieves the sklearn docstring available and does the following:\n        * If length of docstring &lt;= char_lim, then returns the complete docstring\n        * Else, trims the docstring till it encounters a 'Read more in the :ref:'\n        * Or till it encounters a 'Parameters\\n----------\\n'\n        The final string returned is at most of length char_lim with leading and\n        trailing whitespaces removed.\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters for the 'description' field.\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return \"\"\n        try:\n            # trim till 'Read more'\n            pattern = \"Read more in the :ref:\"\n            index = s.index(pattern)\n            s = s[:index]\n            # trimming docstring to be within char_lim\n            if len(s) &gt; char_lim:\n                s = f\"{s[: char_lim - 3]}...\"\n            return s.strip()\n        except ValueError:\n            logger.warning(\n                \"'Read more' not found in descriptions. \"\n                \"Trying to trim till 'Parameters' if available in docstring.\",\n            )\n        try:\n            # if 'Read more' doesn't exist, trim till 'Parameters'\n            pattern = \"Parameters\"\n            index = s.index(match_format(pattern))\n        except ValueError:\n            # returning full docstring\n            logger.warning(\"'Parameters' not found in docstring. Omitting docstring trimming.\")\n            index = len(s)\n        s = s[:index]\n        # trimming docstring to be within char_lim\n        if len(s) &gt; char_lim:\n            s = f\"{s[: char_lim - 3]}...\"\n        return s.strip()\n\n    def _extract_sklearn_parameter_docstring(self, model) -&gt; None | str:\n        \"\"\"Extracts the part of sklearn docstring containing parameter information\n\n        Fetches the entire docstring and trims just the Parameter section.\n        The assumption is that 'Parameters' is the first section in sklearn docstrings,\n        followed by other sections titled 'Attributes', 'See also', 'Note', 'References',\n        appearing in that order if defined.\n        Returns a None if no section with 'Parameters' can be found in the docstring.\n\n        Parameters\n        ----------\n        model : sklearn model\n\n        Returns\n        -------\n        str, or None\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return None\n        try:\n            index1 = s.index(match_format(\"Parameters\"))\n        except ValueError as e:\n            # when sklearn docstring has no 'Parameters' section\n            logger.warning(\"{} {}\".format(match_format(\"Parameters\"), e))\n            return None\n\n        headings = [\"Attributes\", \"Notes\", \"See also\", \"Note\", \"References\"]\n        for h in headings:\n            try:\n                # to find end of Parameters section\n                index2 = s.index(match_format(h))\n                break\n            except ValueError:\n                logger.warning(f\"{h} not available in docstring\")\n                continue\n        else:\n            # in the case only 'Parameters' exist, trim till end of docstring\n            index2 = len(s)\n        s = s[index1:index2]\n        return s.strip()\n\n    def _extract_sklearn_param_info(self, model, char_lim=1024) -&gt; None | dict:\n        \"\"\"Parses parameter type and description from sklearn dosctring\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters string fields.\n\n        Returns\n        -------\n        Dict, or None\n        \"\"\"\n        docstring = self._extract_sklearn_parameter_docstring(model)\n        if docstring is None:\n            # when sklearn docstring has no 'Parameters' section\n            return None\n\n        n = re.compile(\"[.]*\\n\", flags=IGNORECASE)\n        lines = n.split(docstring)\n        p = re.compile(\"[a-z0-9_ ]+ : [a-z0-9_']+[a-z0-9_ ]*\", flags=IGNORECASE)\n        # The above regular expression is designed to detect sklearn parameter names and type\n        # in the format of [variable_name][space]:[space][type]\n        # The expectation is that the parameter description for this detected parameter will\n        # be all the lines in the docstring till the regex finds another parameter match\n\n        # collecting parameters and their descriptions\n        description = []  # type: List\n        for s in lines:\n            param = p.findall(s)\n            if param != []:\n                # a parameter definition is found by regex\n                # creating placeholder when parameter found which will be a list of strings\n                # string descriptions will be appended in subsequent iterations\n                # till another parameter is found and a new placeholder is created\n                placeholder = [\"\"]  # type: List[str]\n                description.append(placeholder)\n            elif len(description) &gt; 0:  # description=[] means no parameters found yet\n                # appending strings to the placeholder created when parameter found\n                description[-1].append(s)\n        for i in range(len(description)):\n            # concatenating parameter description strings\n            description[i] = \"\\n\".join(description[i]).strip()\n            # limiting all parameter descriptions to accepted OpenML string length\n            if len(description[i]) &gt; char_lim:\n                description[i] = f\"{description[i][: char_lim - 3]}...\"\n\n        # collecting parameters and their types\n        parameter_docs = OrderedDict()\n        matches = p.findall(docstring)\n        for i, param in enumerate(matches):\n            key, value = str(param).split(\":\")\n            parameter_docs[key.strip()] = [value.strip(), description[i]]\n\n        # to avoid KeyError for missing parameters\n        param_list_true = list(model.get_params().keys())\n        param_list_found = list(parameter_docs.keys())\n        for param in list(set(param_list_true) - set(param_list_found)):\n            parameter_docs[param] = [None, None]\n\n        return parameter_docs\n\n    def _serialize_model(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Create an OpenMLFlow.\n\n        Calls `sklearn_to_flow` recursively to properly serialize the\n        parameters to strings and the components (other models) to OpenMLFlows.\n\n        Parameters\n        ----------\n        model : sklearn estimator\n\n        Returns\n        -------\n        OpenMLFlow\n\n        \"\"\"\n        # Get all necessary information about the model objects itself\n        (\n            parameters,\n            parameters_meta_info,\n            subcomponents,\n            subcomponents_explicit,\n        ) = self._extract_information_from_model(model)\n\n        # Check that a component does not occur multiple times in a flow as this\n        # is not supported by OpenML\n        self._check_multiple_occurence_of_component_in_flow(model, subcomponents)\n\n        # Create a flow name, which contains all components in brackets, e.g.:\n        # RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),\n        # StandardScaler,AdaBoostClassifier(DecisionTreeClassifier))\n        class_name = model.__module__ + \".\" + model.__class__.__name__\n\n        # will be part of the name (in brackets)\n        sub_components_names = \"\"\n        for key in subcomponents:\n            name_thing = subcomponents[key]\n            if isinstance(name_thing, OpenMLFlow):\n                name = name_thing.name\n            elif (\n                isinstance(name_thing, str)\n                and subcomponents[key] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                name = name_thing\n            else:\n                raise TypeError(type(subcomponents[key]))\n\n            if key in subcomponents_explicit:\n                sub_components_names += \",\" + key + \"=\" + name\n            else:\n                sub_components_names += \",\" + name\n\n        # slice operation on string in order to get rid of leading comma\n        name = f\"{class_name}({sub_components_names[1:]})\" if sub_components_names else class_name\n        short_name = SklearnExtension.trim_flow_name(name)\n\n        # Get the external versions of all sub-components\n        external_version = self._get_external_version_string(model, subcomponents)\n        dependencies = self._get_dependencies()\n        tags = self._get_tags()\n\n        sklearn_description = self._get_sklearn_description(model)\n        return OpenMLFlow(\n            name=name,\n            class_name=class_name,\n            custom_name=short_name,\n            description=sklearn_description,\n            model=model,\n            components=subcomponents,\n            parameters=parameters,\n            parameters_meta_info=parameters_meta_info,\n            external_version=external_version,\n            tags=tags,\n            extension=self,\n            language=\"English\",\n            dependencies=dependencies,\n        )\n\n    def _get_dependencies(self) -&gt; str:\n        return self._min_dependency_str(sklearn.__version__)  # type: ignore\n\n    def _get_tags(self) -&gt; list[str]:\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        sklearn_version_formatted = sklearn_version.replace(\"==\", \"_\")\n        return [\n            \"openml-python\",\n            \"sklearn\",\n            \"scikit-learn\",\n            \"python\",\n            sklearn_version_formatted,\n            # TODO: add more tags based on the scikit-learn\n            # module a flow is in? For example automatically\n            # annotate a class of sklearn.svm.SVC() with the\n            # tag svm?\n        ]\n\n    def _get_external_version_string(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; str:\n        # Create external version string for a flow, given the model and the\n        # already parsed dictionary of sub_components. Retrieves the external\n        # version of all subcomponents, which themselves already contain all\n        # requirements for their subcomponents. The external version string is a\n        # sorted concatenation of all modules which are present in this run.\n\n        external_versions = set()\n\n        # The model is None if the flow is a placeholder flow such as 'passthrough' or 'drop'\n        if model is not None:\n            model_package_name = model.__module__.split(\".\")[0]\n            module = importlib.import_module(model_package_name)\n            model_package_version_number = module.__version__  # type: ignore\n            external_version = self._format_external_version(\n                model_package_name,\n                model_package_version_number,\n            )\n            external_versions.add(external_version)\n\n        openml_version = self._format_external_version(\"openml\", openml.__version__)\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        external_versions.add(openml_version)\n        external_versions.add(sklearn_version)\n        for visitee in sub_components.values():\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                continue\n            for external_version in visitee.external_version.split(\",\"):\n                external_versions.add(external_version)\n        return \",\".join(sorted(external_versions))\n\n    def _check_multiple_occurence_of_component_in_flow(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; None:\n        to_visit_stack: list[OpenMLFlow] = []\n        to_visit_stack.extend(sub_components.values())\n        known_sub_components: set[str] = set()\n\n        while len(to_visit_stack) &gt; 0:\n            visitee = to_visit_stack.pop()\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                known_sub_components.add(visitee)\n            elif visitee.name in known_sub_components:\n                raise ValueError(\n                    f\"Found a second occurence of component {visitee.name} when \"\n                    f\"trying to serialize {model}.\",\n                )\n            else:\n                known_sub_components.add(visitee.name)\n                to_visit_stack.extend(visitee.components.values())\n\n    def _extract_information_from_model(  # noqa: PLR0915, C901, PLR0912\n        self,\n        model: Any,\n    ) -&gt; tuple[\n        OrderedDict[str, str | None],\n        OrderedDict[str, dict | None],\n        OrderedDict[str, OpenMLFlow],\n        set,\n    ]:\n        # This function contains four \"global\" states and is quite long and\n        # complicated. If it gets to complicated to ensure it's correctness,\n        # it would be best to make it a class with the four \"global\" states being\n        # the class attributes and the if/elif/else in the for-loop calls to\n        # separate class methods\n\n        # stores all entities that should become subcomponents\n        sub_components = OrderedDict()  # type: OrderedDict[str, OpenMLFlow]\n        # stores the keys of all subcomponents that should become\n        sub_components_explicit = set()\n        parameters: OrderedDict[str, str | None] = OrderedDict()\n        parameters_meta_info: OrderedDict[str, dict | None] = OrderedDict()\n        parameters_docs = self._extract_sklearn_param_info(model)\n\n        model_parameters = model.get_params(deep=False)\n        for k, v in sorted(model_parameters.items(), key=lambda t: t[0]):\n            rval = self._serialize_sklearn(v, model)\n\n            def flatten_all(list_):\n                \"\"\"Flattens arbitrary depth lists of lists (e.g. [[1,2],[3,[1]]] -&gt; [1,2,3,1]).\"\"\"\n                for el in list_:\n                    if isinstance(el, (list, tuple)) and len(el) &gt; 0:\n                        yield from flatten_all(el)\n                    else:\n                        yield el\n\n            # In case rval is a list of lists (or tuples), we need to identify two situations:\n            # - sklearn pipeline steps, feature union or base classifiers in voting classifier.\n            #   They look like e.g. [(\"imputer\", Imputer()), (\"classifier\", SVC())]\n            # - a list of lists with simple types (e.g. int or str), such as for an OrdinalEncoder\n            #   where all possible values for each feature are described: [[0,1,2], [1,2,5]]\n            is_non_empty_list_of_lists_with_same_type = (\n                isinstance(rval, (list, tuple))\n                and len(rval) &gt; 0\n                and isinstance(rval[0], (list, tuple))\n                and all(isinstance(rval_i, type(rval[0])) for rval_i in rval)\n            )\n\n            # Check that all list elements are of simple types.\n            nested_list_of_simple_types = (\n                is_non_empty_list_of_lists_with_same_type\n                and all(isinstance(el, SIMPLE_TYPES) for el in flatten_all(rval))\n                and all(\n                    len(rv) in (2, 3) and rv[1] not in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    for rv in rval\n                )\n            )\n\n            if is_non_empty_list_of_lists_with_same_type and not nested_list_of_simple_types:\n                # If a list of lists is identified that include 'non-simple' types (e.g. objects),\n                # we assume they are steps in a pipeline, feature union, or base classifiers in\n                # a voting classifier.\n                parameter_value = []  # type: List\n                reserved_keywords = set(model.get_params(deep=False).keys())\n\n                for sub_component_tuple in rval:\n                    identifier = sub_component_tuple[0]\n                    sub_component = sub_component_tuple[1]\n                    sub_component_type = type(sub_component_tuple)\n                    if not 2 &lt;= len(sub_component_tuple) &lt;= 3:\n                        # length 2 is for {VotingClassifier.estimators,\n                        # Pipeline.steps, FeatureUnion.transformer_list}\n                        # length 3 is for ColumnTransformer\n                        msg = \"Length of tuple of type {} does not match assumptions\".format(\n                            sub_component_type,\n                        )\n                        raise ValueError(msg)\n\n                    if isinstance(sub_component, str):\n                        if sub_component not in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                            msg = (\n                                \"Second item of tuple does not match assumptions. \"\n                                \"If string, can be only 'drop' or 'passthrough' but\"\n                                \"got %s\" % sub_component\n                            )\n                            raise ValueError(msg)\n                    elif sub_component is None:\n                        msg = (\n                            \"Cannot serialize objects of None type. Please use a valid \"\n                            \"placeholder for None. Note that empty sklearn estimators can be \"\n                            \"replaced with 'drop' or 'passthrough'.\"\n                        )\n                        raise ValueError(msg)\n                    elif not isinstance(sub_component, OpenMLFlow):\n                        msg = (\n                            \"Second item of tuple does not match assumptions. \"\n                            \"Expected OpenMLFlow, got %s\" % type(sub_component)\n                        )\n                        raise TypeError(msg)\n\n                    if identifier in reserved_keywords:\n                        parent_model = f\"{model.__module__}.{model.__class__.__name__}\"\n                        msg = \"Found element shadowing official \" \"parameter for {}: {}\".format(\n                            parent_model,\n                            identifier,\n                        )\n                        raise PyOpenMLError(msg)\n\n                    # when deserializing the parameter\n                    sub_components_explicit.add(identifier)\n                    if isinstance(sub_component, str):\n                        external_version = self._get_external_version_string(None, {})\n                        dependencies = self._get_dependencies()\n                        tags = self._get_tags()\n\n                        sub_components[identifier] = OpenMLFlow(\n                            name=sub_component,\n                            description=\"Placeholder flow for scikit-learn's string pipeline \"\n                            \"members\",\n                            components=OrderedDict(),\n                            parameters=OrderedDict(),\n                            parameters_meta_info=OrderedDict(),\n                            external_version=external_version,\n                            tags=tags,\n                            language=\"English\",\n                            dependencies=dependencies,\n                            model=None,\n                        )\n                        component_reference: OrderedDict[str, str | dict] = OrderedDict()\n                        component_reference[\n                            \"oml-python:serialized_object\"\n                        ] = COMPOSITION_STEP_CONSTANT\n                        cr_value: dict[str, Any] = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    else:\n                        sub_components[identifier] = sub_component\n                        component_reference = OrderedDict()\n                        component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                        cr_value = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    parameter_value.append(component_reference)\n\n                # Here (and in the elif and else branch below) are the only\n                # places where we encode a value as json to make sure that all\n                # parameter values still have the same type after\n                # deserialization\n                if isinstance(rval, tuple):\n                    parameter_json = json.dumps(tuple(parameter_value))\n                else:\n                    parameter_json = json.dumps(parameter_value)\n                parameters[k] = parameter_json\n\n            elif isinstance(rval, OpenMLFlow):\n                # A subcomponent, for example the base model in\n                # AdaBoostClassifier\n                sub_components[k] = rval\n                sub_components_explicit.add(k)\n                component_reference = OrderedDict()\n                component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                cr_value = OrderedDict()\n                cr_value[\"key\"] = k\n                cr_value[\"step_name\"] = None\n                component_reference[\"value\"] = cr_value\n                cr = self._serialize_sklearn(component_reference, model)\n                parameters[k] = json.dumps(cr)\n\n            elif not (hasattr(rval, \"__len__\") and len(rval) == 0):\n                rval = json.dumps(rval)\n                parameters[k] = rval\n            # a regular hyperparameter\n            else:\n                parameters[k] = None\n\n            if parameters_docs is not None:\n                data_type, description = parameters_docs[k]\n                parameters_meta_info[k] = OrderedDict(\n                    ((\"description\", description), (\"data_type\", data_type)),\n                )\n            else:\n                parameters_meta_info[k] = OrderedDict(((\"description\", None), (\"data_type\", None)))\n\n        return parameters, parameters_meta_info, sub_components, sub_components_explicit\n\n    def _get_fn_arguments_with_defaults(self, fn_name: Callable) -&gt; tuple[dict, set]:\n        \"\"\"\n        Returns\n        -------\n            i) a dict with all parameter names that have a default value, and\n            ii) a set with all parameter names that do not have a default\n\n        Parameters\n        ----------\n        fn_name : callable\n            The function of which we want to obtain the defaults\n\n        Returns\n        -------\n        params_with_defaults: dict\n            a dict mapping parameter name to the default value\n        params_without_defaults: set\n            a set with all parameters that do not have a default value\n        \"\"\"\n        # parameters with defaults are optional, all others are required.\n        parameters = inspect.signature(fn_name).parameters\n        required_params = set()\n        optional_params = {}\n        for param in parameters:\n            parameter = parameters.get(param)\n            default_val = parameter.default  # type: ignore\n            if default_val is inspect.Signature.empty:\n                required_params.add(param)\n            else:\n                optional_params[param] = default_val\n        return optional_params, required_params\n\n    def _deserialize_model(\n        self,\n        flow: OpenMLFlow,\n        keep_defaults: bool,  # noqa: FBT001\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        logger.info(\"-{} deserialize {}\".format(\"-\" * recursion_depth, flow.name))\n        model_name = flow.class_name\n        self._check_dependencies(flow.dependencies, strict_version=strict_version)\n\n        parameters = flow.parameters\n        components = flow.components\n        parameter_dict: dict[str, Any] = OrderedDict()\n\n        # Do a shallow copy of the components dictionary so we can remove the\n        # components from this copy once we added them into the pipeline. This\n        # allows us to not consider them any more when looping over the\n        # components, but keeping the dictionary of components untouched in the\n        # original components dictionary.\n        components_ = copy.copy(components)\n\n        for name in parameters:\n            value = parameters.get(name)\n            logger.info(\n                \"--{} flow_parameter={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                components=components_,\n                initialize_with_defaults=keep_defaults,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        for name in components:\n            if name in parameter_dict:\n                continue\n            if name not in components_:\n                continue\n            value = components[name]\n            logger.info(\n                \"--{} flow_component={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        if model_name is None and flow.name in SKLEARN_PIPELINE_STRING_COMPONENTS:\n            return flow.name\n\n        assert model_name is not None\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n\n        if keep_defaults:\n            # obtain all params with a default\n            param_defaults, _ = self._get_fn_arguments_with_defaults(model_class.__init__)\n\n            # delete the params that have a default from the dict,\n            # so they get initialized with their default value\n            # except [...]\n            for param in param_defaults:\n                # [...] the ones that also have a key in the components dict.\n                # As OpenML stores different flows for ensembles with different\n                # (base-)components, in OpenML terms, these are not considered\n                # hyperparameters but rather constants (i.e., changing them would\n                # result in a different flow)\n                if param not in components:\n                    del parameter_dict[param]\n\n        return model_class(**parameter_dict)\n\n    def _check_dependencies(\n        self,\n        dependencies: str,\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        if not dependencies:\n            return\n\n        dependencies_list = dependencies.split(\"\\n\")\n        for dependency_string in dependencies_list:\n            match = DEPENDENCIES_PATTERN.match(dependency_string)\n            if not match:\n                raise ValueError(\"Cannot parse dependency %s\" % dependency_string)\n\n            dependency_name = match.group(\"name\")\n            operation = match.group(\"operation\")\n            version = match.group(\"version\")\n\n            module = importlib.import_module(dependency_name)\n            required_version = LooseVersion(version)\n            installed_version = LooseVersion(module.__version__)  # type: ignore\n\n            if operation == \"==\":\n                check = required_version == installed_version\n            elif operation == \"&gt;\":\n                check = installed_version &gt; required_version\n            elif operation == \"&gt;=\":\n                check = (\n                    installed_version &gt; required_version or installed_version == required_version\n                )\n            else:\n                raise NotImplementedError(\"operation '%s' is not supported\" % operation)\n            message = (\n                \"Trying to deserialize a model with dependency \"\n                f\"{dependency_string} not satisfied.\"\n            )\n            if not check:\n                if strict_version:\n                    raise ValueError(message)\n\n                warnings.warn(message, category=UserWarning, stacklevel=2)\n\n    def _serialize_type(self, o: Any) -&gt; OrderedDict[str, str]:\n        mapping = {\n            float: \"float\",\n            np.float32: \"np.float32\",\n            np.float64: \"np.float64\",\n            int: \"int\",\n            np.int32: \"np.int32\",\n            np.int64: \"np.int64\",\n        }\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[float] = \"np.float\"\n            mapping[int] = \"np.int\"\n\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"type\"\n        ret[\"value\"] = mapping[o]\n        return ret\n\n    def _deserialize_type(self, o: str) -&gt; Any:\n        mapping = {\n            \"float\": float,\n            \"np.float32\": np.float32,\n            \"np.float64\": np.float64,\n            \"int\": int,\n            \"np.int32\": np.int32,\n            \"np.int64\": np.int64,\n        }\n\n        # TODO(eddiebergman): Might be able to remove this\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[\"np.float\"] = np.float  # type: ignore # noqa: NPY001\n            mapping[\"np.int\"] = np.int  # type: ignore # noqa: NPY001\n\n        return mapping[o]\n\n    def _serialize_rv_frozen(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        args = o.args\n        kwds = o.kwds\n        a = o.a\n        b = o.b\n        dist = o.dist.__class__.__module__ + \".\" + o.dist.__class__.__name__\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n        ret[\"oml-python:serialized_object\"] = \"rv_frozen\"\n        ret[\"value\"] = OrderedDict(\n            ((\"dist\", dist), (\"a\", a), (\"b\", b), (\"args\", args), (\"kwds\", kwds)),\n        )\n        return ret\n\n    def _deserialize_rv_frozen(self, o: OrderedDict[str, str]) -&gt; Any:\n        args = o[\"args\"]\n        kwds = o[\"kwds\"]\n        a = o[\"a\"]\n        b = o[\"b\"]\n        dist_name = o[\"dist\"]\n\n        module_name = dist_name.rsplit(\".\", 1)\n        try:\n            rv_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        except AttributeError as e:\n            _tb = traceback.format_exc()\n            warnings.warn(\n                f\"Cannot create model {dist_name} for flow. Reason is from error {type(e)}:{e}\"\n                f\"\\nTraceback: {_tb}\",\n                RuntimeWarning,\n                stacklevel=2,\n            )\n            return None\n\n        dist = scipy.stats.distributions.rv_frozen(rv_class(), *args, **kwds)  # type: ignore\n        dist.a = a\n        dist.b = b\n\n        return dist\n\n    def _serialize_function(self, o: Callable) -&gt; OrderedDict[str, str]:\n        name = o.__module__ + \".\" + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"function\"\n        ret[\"value\"] = name\n        return ret\n\n    def _deserialize_function(self, name: str) -&gt; Callable:\n        module_name = name.rsplit(\".\", 1)\n        return getattr(importlib.import_module(module_name[0]), module_name[1])\n\n    def _serialize_cross_validator(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n\n        parameters = OrderedDict()  # type: 'OrderedDict[str, Any]'\n\n        # XXX this is copied from sklearn.model_selection._split\n        cls = o.__class__\n        init = getattr(cls.__init__, \"deprecated_original\", cls.__init__)\n        # Ignore varargs, kw and default values and pop self\n        init_signature = inspect.signature(init)  # type: ignore\n        # Consider the constructor parameters excluding 'self'\n        if init is object.__init__:\n            args = []  # type: List\n        else:\n            args = sorted(\n                [\n                    p.name\n                    for p in init_signature.parameters.values()\n                    if p.name != \"self\" and p.kind != p.VAR_KEYWORD\n                ],\n            )\n\n        for key in args:\n            # We need deprecation warnings to always be on in order to\n            # catch deprecated param values.\n            # This is set in utils/__init__.py but it gets overwritten\n            # when running under python3 somehow.\n            with warnings.catch_warnings(record=True) as w:\n                warnings.simplefilter(\"always\", DeprecationWarning)\n                value = getattr(o, key, None)\n                if w is not None and len(w) and w[0].category == DeprecationWarning:\n                    # if the parameter is deprecated, don't show it\n                    continue\n\n            if not (isinstance(value, Sized) and len(value) == 0):\n                value = json.dumps(value)\n                parameters[key] = value\n            else:\n                parameters[key] = None\n\n        ret[\"oml-python:serialized_object\"] = \"cv_object\"\n        name = o.__module__ + \".\" + o.__class__.__name__\n        value = OrderedDict([(\"name\", name), (\"parameters\", parameters)])\n        ret[\"value\"] = value\n\n        return ret\n\n    def _deserialize_cross_validator(\n        self,\n        value: OrderedDict[str, Any],\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        model_name = value[\"name\"]\n        parameters = value[\"parameters\"]\n\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        for parameter in parameters:\n            parameters[parameter] = self._deserialize_sklearn(\n                parameters[parameter],\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n        return model_class(**parameters)\n\n    def _format_external_version(\n        self,\n        model_package_name: str,\n        model_package_version_number: str,\n    ) -&gt; str:\n        return f\"{model_package_name}=={model_package_version_number}\"\n\n    @staticmethod\n    def _get_parameter_values_recursive(\n        param_grid: dict | list[dict],\n        parameter_name: str,\n    ) -&gt; list[Any]:\n        \"\"\"\n        Returns a list of values for a given hyperparameter, encountered\n        recursively throughout the flow. (e.g., n_jobs can be defined\n        for various flows)\n\n        Parameters\n        ----------\n        param_grid: Union[Dict, List[Dict]]\n            Dict mapping from hyperparameter list to value, to a list of\n            such dicts\n\n        parameter_name: str\n            The hyperparameter that needs to be inspected\n\n        Returns\n        -------\n        List\n            A list of all values of hyperparameters with this name\n        \"\"\"\n        if isinstance(param_grid, dict):\n            return [\n                value\n                for param, value in param_grid.items()\n                if param.split(\"__\")[-1] == parameter_name\n            ]\n\n        if isinstance(param_grid, list):\n            result = []\n            for sub_grid in param_grid:\n                result.extend(\n                    SklearnExtension._get_parameter_values_recursive(sub_grid, parameter_name),\n                )\n            return result\n\n        raise ValueError(\"Param_grid should either be a dict or list of dicts\")\n\n    def _prevent_optimize_n_jobs(self, model):\n        \"\"\"\n        Ensures that HPO classes will not optimize the n_jobs hyperparameter\n\n        Parameters\n        ----------\n        model:\n            The model that will be fitted\n        \"\"\"\n        if self._is_hpo_class(model):\n            if isinstance(model, sklearn.model_selection.GridSearchCV):\n                param_distributions = model.param_grid\n            elif isinstance(model, sklearn.model_selection.RandomizedSearchCV):\n                param_distributions = model.param_distributions\n            else:\n                if hasattr(model, \"param_distributions\"):\n                    param_distributions = model.param_distributions\n                else:\n                    raise AttributeError(\n                        \"Using subclass BaseSearchCV other than \"\n                        \"{GridSearchCV, RandomizedSearchCV}. \"\n                        \"Could not find attribute \"\n                        \"param_distributions.\",\n                    )\n                logger.warning(\n                    \"Warning! Using subclass BaseSearchCV other than \"\n                    \"{GridSearchCV, RandomizedSearchCV}. \"\n                    \"Should implement param check. \",\n                )\n            n_jobs_vals = SklearnExtension._get_parameter_values_recursive(\n                param_distributions,\n                \"n_jobs\",\n            )\n            if len(n_jobs_vals) &gt; 0:\n                raise PyOpenMLError(\n                    \"openml-python should not be used to \" \"optimize the n_jobs parameter.\",\n                )\n\n    ################################################################################################\n    # Methods for performing runs with extension modules\n\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is a scikit-learn estimator.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        o = model\n        return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n\n    def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n        \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n        model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Models that are already seeded will maintain the seed. In this case,\n        only integer seeds are allowed (An exception is raised when a RandomState was used as\n        seed).\n\n        Parameters\n        ----------\n        model : sklearn model\n            The model to be seeded\n        seed : int\n            The seed to initialize the RandomState with. Unseeded subcomponents\n            will be seeded with a random number from the RandomState.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        def _seed_current_object(current_value):\n            if isinstance(current_value, int):  # acceptable behaviour\n                return False\n\n            if isinstance(current_value, np.random.RandomState):\n                raise ValueError(\n                    \"Models initialized with a RandomState object are not \"\n                    \"supported. Please seed with an integer. \",\n                )\n\n            if current_value is not None:\n                raise ValueError(\n                    \"Models should be seeded with int or None (this should never \" \"happen). \",\n                )\n\n            return True\n\n        rs = np.random.RandomState(seed)\n        model_params = model.get_params()\n        random_states = {}\n        for param_name in sorted(model_params):\n            if \"random_state\" in param_name:\n                current_value = model_params[param_name]\n                # important to draw the value at this point (and not in the if\n                # statement) this way we guarantee that if a different set of\n                # subflows is seeded, the same number of the random generator is\n                # used\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    random_states[param_name] = new_value\n\n            # Also seed CV objects!\n            elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n                if not hasattr(model_params[param_name], \"random_state\"):\n                    continue\n\n                current_value = model_params[param_name].random_state\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    model_params[param_name].random_state = new_value\n\n        model.set_params(**random_states)\n        return model\n\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        from sklearn.exceptions import NotFittedError\n        from sklearn.utils.validation import check_is_fitted\n\n        try:\n            # check if model is fitted\n            check_is_fitted(model)\n\n            # Creating random dummy data of arbitrary size\n            dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n            # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n            # robust check that works across sklearn versions and models. Internally, 'predict'\n            # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n            # assured check than explicit calls to 'check_is_fitted'\n            model.predict(dummy_data)\n            # Will reach here if the model was fit on a dataset with 3 features\n            return True\n        except NotFittedError:  # needs to be the first exception to be caught\n            # Model is not fitted, as is required\n            return False\n        except ValueError:\n            # Will reach here if the model was fit on a dataset with more or less than 3 features\n            return True\n\n    def _run_model_on_fold(  # noqa: PLR0915, PLR0913, C901, PLR0912\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame | None = None,\n    ) -&gt; tuple[\n        np.ndarray,\n        pd.DataFrame | None,\n        OrderedDict[str, float],\n        OpenMLRunTrace | None,\n    ]:\n        \"\"\"Run a model on a repeat,fold,subsample triplet of the task and return prediction\n        information.\n\n        Furthermore, it will measure run time measures in case multi-core behaviour allows this.\n        * exact user cpu time will be measured if the number of cores is set (recursive throughout\n        the model) exactly to 1\n        * wall clock time will be measured if the number of cores is set (recursive throughout the\n        model) to any given number (but not when it is set to -1)\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        run_task_get_arff_content. Do not use this function unless you know what you are doing.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        pred_y : np.ndarray\n            Predictions on the training/test set, depending on the task type.\n            For supervised tasks, predictions are on the test set.\n            For unsupervised tasks, predictions are on the training set.\n        proba_y : pd.DataFrame, optional\n            Predicted probabilities for the test set.\n            None, if task is not Classification or Learning Curve prediction.\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : OpenMLRunTrace, optional\n            arff trace object from a fitted model and the trace content obtained by\n            repeatedly calling ``run_model_on_task``\n        \"\"\"\n\n        def _prediction_to_probabilities(\n            y: np.ndarray | list,\n            model_classes: list[Any],\n            class_labels: list[str] | None,\n        ) -&gt; pd.DataFrame:\n            \"\"\"Transforms predicted probabilities to match with OpenML class indices.\n\n            Parameters\n            ----------\n            y : np.ndarray\n                Predicted probabilities (possibly omitting classes if they were not present in the\n                training data).\n            model_classes : list\n                List of classes known_predicted by the model, ordered by their index.\n            class_labels : list\n                List of classes as stored in the task object fetched from server.\n\n            Returns\n            -------\n            pd.DataFrame\n            \"\"\"\n            if class_labels is None:\n                raise ValueError(\"The task has no class labels\")\n\n            if isinstance(y_train, np.ndarray) and isinstance(class_labels[0], str):\n                # mapping (decoding) the predictions to the categories\n                # creating a separate copy to not change the expected pred_y type\n                y = [class_labels[pred] for pred in y]  # list or numpy array of predictions\n\n            # model_classes: sklearn classifier mapping from original array id to\n            # prediction index id\n            if not isinstance(model_classes, list):\n                raise ValueError(\"please convert model classes to list prior to calling this fn\")\n\n            # DataFrame allows more accurate mapping of classes as column names\n            result = pd.DataFrame(\n                0,\n                index=np.arange(len(y)),\n                columns=model_classes,\n                dtype=np.float32,\n            )\n            for obs, prediction in enumerate(y):\n                result.loc[obs, prediction] = 1.0\n            return result\n\n        if isinstance(task, OpenMLSupervisedTask):\n            if y_train is None:\n                raise TypeError(\"argument y_train must not be of type None\")\n            if X_test is None:\n                raise TypeError(\"argument X_test must not be of type None\")\n\n        model_copy = sklearn.base.clone(model, safe=True)\n        # sanity check: prohibit users from optimizing n_jobs\n        self._prevent_optimize_n_jobs(model_copy)\n        # measures and stores runtimes\n        user_defined_measures = OrderedDict()  # type: 'OrderedDict[str, float]'\n        try:\n            # for measuring runtime. Only available since Python 3.3\n            modelfit_start_cputime = time.process_time()\n            modelfit_start_walltime = time.time()\n\n            if isinstance(task, OpenMLSupervisedTask):\n                model_copy.fit(X_train, y_train)  # type: ignore\n            elif isinstance(task, OpenMLClusteringTask):\n                model_copy.fit(X_train)  # type: ignore\n\n            modelfit_dur_cputime = (time.process_time() - modelfit_start_cputime) * 1000\n            modelfit_dur_walltime = (time.time() - modelfit_start_walltime) * 1000\n\n            user_defined_measures[\"usercpu_time_millis_training\"] = modelfit_dur_cputime\n            refit_time = model_copy.refit_time_ * 1000 if hasattr(model_copy, \"refit_time_\") else 0  # type: ignore\n            user_defined_measures[\"wall_clock_time_millis_training\"] = modelfit_dur_walltime\n\n        except AttributeError as e:\n            # typically happens when training a regressor on classification task\n            raise PyOpenMLError(str(e)) from e\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            # search for model classes_ (might differ depending on modeltype)\n            # first, pipelines are a special case (these don't have a classes_\n            # object, but rather borrows it from the last step. We do this manually,\n            # because of the BaseSearch check)\n            if isinstance(model_copy, sklearn.pipeline.Pipeline):\n                used_estimator = model_copy.steps[-1][-1]\n            else:\n                used_estimator = model_copy\n\n            if self._is_hpo_class(used_estimator):\n                model_classes = used_estimator.best_estimator_.classes_\n            else:\n                model_classes = used_estimator.classes_\n\n            if not isinstance(model_classes, list):\n                model_classes = model_classes.tolist()\n\n            # to handle the case when dataset is numpy and categories are encoded\n            # however the class labels stored in task are still categories\n            if isinstance(y_train, np.ndarray) and isinstance(\n                cast(List, task.class_labels)[0],\n                str,\n            ):\n                model_classes = [cast(List[str], task.class_labels)[i] for i in model_classes]\n\n        modelpredict_start_cputime = time.process_time()\n        modelpredict_start_walltime = time.time()\n\n        # In supervised learning this returns the predictions for Y, in clustering\n        # it returns the clusters\n        if isinstance(task, OpenMLSupervisedTask):\n            pred_y = model_copy.predict(X_test)\n        elif isinstance(task, OpenMLClusteringTask):\n            pred_y = model_copy.predict(X_train)\n        else:\n            raise ValueError(task)\n\n        modelpredict_duration_cputime = (time.process_time() - modelpredict_start_cputime) * 1000\n        user_defined_measures[\"usercpu_time_millis_testing\"] = modelpredict_duration_cputime\n        user_defined_measures[\"usercpu_time_millis\"] = (\n            modelfit_dur_cputime + modelpredict_duration_cputime\n        )\n        modelpredict_duration_walltime = (time.time() - modelpredict_start_walltime) * 1000\n        user_defined_measures[\"wall_clock_time_millis_testing\"] = modelpredict_duration_walltime\n        user_defined_measures[\"wall_clock_time_millis\"] = (\n            modelfit_dur_walltime + modelpredict_duration_walltime + refit_time\n        )\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            try:\n                proba_y = model_copy.predict_proba(X_test)\n                proba_y = pd.DataFrame(proba_y, columns=model_classes)  # handles X_test as numpy\n            except AttributeError:  # predict_proba is not available when probability=False\n                proba_y = _prediction_to_probabilities(pred_y, model_classes, task.class_labels)\n\n            if task.class_labels is not None:\n                if proba_y.shape[1] != len(task.class_labels):\n                    # Remap the probabilities in case there was a class missing\n                    # at training time. By default, the classification targets\n                    # are mapped to be zero-based indices to the actual classes.\n                    # Therefore, the model_classes contain the correct indices to\n                    # the correct probability array. Example:\n                    # classes in the dataset: 0, 1, 2, 3, 4, 5\n                    # classes in the training set: 0, 1, 2, 4, 5\n                    # then we need to add a column full of zeros into the probabilities\n                    # for class 3 because the rest of the library expects that the\n                    # probabilities are ordered the same way as the classes are ordered).\n                    message = \"Estimator only predicted for {}/{} classes!\".format(\n                        proba_y.shape[1],\n                        len(task.class_labels),\n                    )\n                    warnings.warn(message, stacklevel=2)\n                    openml.config.logger.warning(message)\n\n                    for _i, col in enumerate(task.class_labels):\n                        # adding missing columns with 0 probability\n                        if col not in model_classes:\n                            proba_y[col] = 0\n                    # We re-order the columns to move possibly added missing columns into place.\n                    proba_y = proba_y[task.class_labels]\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n            if not np.all(set(proba_y.columns) == set(task.class_labels)):\n                missing_cols = list(set(task.class_labels) - set(proba_y.columns))\n                raise ValueError(\"Predicted probabilities missing for the columns: \", missing_cols)\n\n        elif isinstance(task, (OpenMLRegressionTask, OpenMLClusteringTask)):\n            proba_y = None\n        else:\n            raise TypeError(type(task))\n\n        if self._is_hpo_class(model_copy):\n            trace_data = self._extract_trace_data(model_copy, rep_no, fold_no)\n            trace: OpenMLRunTrace | None = self._obtain_arff_trace(\n                model_copy,\n                trace_data,\n            )\n        else:\n            trace = None\n\n        return pred_y, proba_y, user_defined_measures, trace\n\n    def obtain_parameter_values(  # noqa: C901, PLR0915\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n        openml.flows.functions._check_flow_for_server_id(flow)\n\n        def get_flow_dict(_flow):\n            flow_map = {_flow.name: _flow.flow_id}\n            for subflow in _flow.components:\n                flow_map.update(get_flow_dict(_flow.components[subflow]))\n            return flow_map\n\n        def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n            _flow,\n            _flow_dict,\n            component_model,\n            _main_call=False,  # noqa: FBT002\n            main_id=None,\n        ):\n            def is_subcomponent_specification(values):\n                # checks whether the current value can be a specification of\n                # subcomponents, as for example the value for steps parameter\n                # (in Pipeline) or transformers parameter (in\n                # ColumnTransformer).\n                return (\n                    # Specification requires list/tuple of list/tuple with\n                    # at least length 2.\n                    isinstance(values, (tuple, list))\n                    and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                    # And each component needs to be a flow or interpretable string\n                    and all(\n                        isinstance(item[1], openml.flows.OpenMLFlow)\n                        or (\n                            isinstance(item[1], str)\n                            and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        )\n                        for item in values\n                    )\n                )\n\n            # _flow is openml flow object, _param dict maps from flow name to flow\n            # id for the main call, the param dict can be overridden (useful for\n            # unit tests / sentinels) this way, for flows without subflows we do\n            # not have to rely on _flow_dict\n            exp_parameters = set(_flow.parameters)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_parameters = set()\n            else:\n                model_parameters = set(component_model.get_params(deep=False))\n            if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n                flow_params = sorted(exp_parameters)\n                model_params = sorted(model_parameters)\n                raise ValueError(\n                    \"Parameters of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow parameters: \"\n                    f\"{flow_params}\\nmodel parameters: {model_params}\",\n                )\n            exp_components = set(_flow.components)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_components = set()\n            else:\n                _ = set(component_model.get_params(deep=False))\n                model_components = {\n                    mp\n                    for mp in component_model.get_params(deep=True)\n                    if \"__\" not in mp and mp not in _\n                }\n            if len(exp_components.symmetric_difference(model_components)) != 0:\n                is_problem = True\n                if len(exp_components - model_components) &gt; 0:\n                    # If an expected component is not returned as a component by get_params(),\n                    # this means that it is also a parameter -&gt; we need to check that this is\n                    # actually the case\n                    difference = exp_components - model_components\n                    component_in_model_parameters = []\n                    for component in difference:\n                        if component in model_parameters:\n                            component_in_model_parameters.append(True)\n                        else:\n                            component_in_model_parameters.append(False)\n                    is_problem = not all(component_in_model_parameters)\n                if is_problem:\n                    flow_components = sorted(exp_components)\n                    model_components = sorted(model_components)\n                    raise ValueError(\n                        \"Subcomponents of the model do not match the \"\n                        \"parameters expected by the \"\n                        \"flow:\\nexpected flow subcomponents: \"\n                        f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                    )\n\n            _params = []\n            for _param_name in _flow.parameters:\n                _current = OrderedDict()\n                _current[\"oml:name\"] = _param_name\n\n                current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n                # Try to filter out components (a.k.a. subflows) which are\n                # handled further down in the code (by recursively calling\n                # this function)!\n                if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                    continue\n\n                if is_subcomponent_specification(current_param_values):\n                    # complex parameter value, with subcomponents\n                    parsed_values = []\n                    for subcomponent in current_param_values:\n                        # scikit-learn stores usually tuples in the form\n                        # (name (str), subcomponent (mixed), argument\n                        # (mixed)). OpenML replaces the subcomponent by an\n                        # OpenMLFlow object.\n                        if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                            raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                        subcomponent_identifier = subcomponent[0]\n                        subcomponent_flow = subcomponent[1]\n                        if not isinstance(subcomponent_identifier, str):\n                            raise TypeError(\n                                \"Subcomponent identifier should be of type string, \"\n                                f\"but is {type(subcomponent_identifier)}\",\n                            )\n                        if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                            if (\n                                isinstance(subcomponent_flow, str)\n                                and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                            ):\n                                pass\n                            else:\n                                raise TypeError(\n                                    \"Subcomponent flow should be of type flow, but is {}\".format(\n                                        type(subcomponent_flow),\n                                    ),\n                                )\n\n                        current = {\n                            \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                            \"value\": {\n                                \"key\": subcomponent_identifier,\n                                \"step_name\": subcomponent_identifier,\n                            },\n                        }\n                        if len(subcomponent) == 3:\n                            if not isinstance(subcomponent[2], list) and not isinstance(\n                                subcomponent[2],\n                                OrderedDict,\n                            ):\n                                raise TypeError(\n                                    \"Subcomponent argument should be list or OrderedDict\",\n                                )\n                            current[\"value\"][\"argument_1\"] = subcomponent[2]\n                        parsed_values.append(current)\n                    parsed_values = json.dumps(parsed_values)\n                else:\n                    # vanilla parameter value\n                    parsed_values = json.dumps(current_param_values)\n\n                _current[\"oml:value\"] = parsed_values\n                if _main_call:\n                    _current[\"oml:component\"] = main_id\n                else:\n                    _current[\"oml:component\"] = _flow_dict[_flow.name]\n                _params.append(_current)\n\n            for _identifier in _flow.components:\n                subcomponent_model = component_model.get_params()[_identifier]\n                _params.extend(\n                    extract_parameters(\n                        _flow.components[_identifier],\n                        _flow_dict,\n                        subcomponent_model,\n                    ),\n                )\n            return _params\n\n        flow_dict = get_flow_dict(flow)\n        model = model if model is not None else flow.model\n        return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n\n    def _openml_param_name_to_sklearn(\n        self,\n        openml_parameter: openml.setups.OpenMLParameter,\n        flow: OpenMLFlow,\n    ) -&gt; str:\n        \"\"\"\n        Converts the name of an OpenMLParameter into the sklean name, given a flow.\n\n        Parameters\n        ----------\n        openml_parameter: OpenMLParameter\n            The parameter under consideration\n\n        flow: OpenMLFlow\n            The flow that provides context.\n\n        Returns\n        -------\n        sklearn_parameter_name: str\n            The name the parameter will have once used in scikit-learn\n        \"\"\"\n        if not isinstance(openml_parameter, openml.setups.OpenMLParameter):\n            raise ValueError(\"openml_parameter should be an instance of OpenMLParameter\")\n        if not isinstance(flow, OpenMLFlow):\n            raise ValueError(\"flow should be an instance of OpenMLFlow\")\n\n        flow_structure = flow.get_structure(\"name\")\n        if openml_parameter.flow_name not in flow_structure:\n            raise ValueError(\"Obtained OpenMLParameter and OpenMLFlow do not correspond. \")\n        name = openml_parameter.flow_name  # for PEP8\n        return \"__\".join(flow_structure[name] + [openml_parameter.parameter_name])\n\n    ################################################################################################\n    # Methods for hyperparameter optimization\n\n    def _is_hpo_class(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the model performs hyperparameter optimization.\n\n        Used to check whether an optimization trace can be extracted from the model after\n        running it.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.model_selection._search.BaseSearchCV)\n\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n        optimization model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        base_estimator = model.estimator\n        base_estimator.set_params(**trace_iteration.get_parameters())\n        return base_estimator\n\n    def _extract_trace_data(self, model, rep_no, fold_no):\n        \"\"\"Extracts data from a machine learning model's cross-validation results\n        and creates an ARFF (Attribute-Relation File Format) trace.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n        rep_no : int\n            The repetition number.\n        fold_no : int\n            The fold number.\n\n        Returns\n        -------\n        A list of ARFF tracecontent.\n        \"\"\"\n        arff_tracecontent = []\n        for itt_no in range(len(model.cv_results_[\"mean_test_score\"])):\n            # we use the string values for True and False, as it is defined in\n            # this way by the OpenML server\n            selected = \"false\"\n            if itt_no == model.best_index_:\n                selected = \"true\"\n            test_score = model.cv_results_[\"mean_test_score\"][itt_no]\n            arff_line = [rep_no, fold_no, itt_no, test_score, selected]\n            for key in model.cv_results_:\n                if key.startswith(\"param_\"):\n                    value = model.cv_results_[key][itt_no]\n                    # Built-in serializer does not convert all numpy types,\n                    # these methods convert them to built-in types instead.\n                    if isinstance(value, np.generic):\n                        # For scalars it actually returns scalars, not a list\n                        value = value.tolist()\n                    serialized_value = json.dumps(value) if value is not np.ma.masked else np.nan\n                    arff_line.append(serialized_value)\n            arff_tracecontent.append(arff_line)\n        return arff_tracecontent\n\n    def _obtain_arff_trace(\n        self,\n        model: Any,\n        trace_content: list,\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Create arff trace object from a fitted model and the trace content obtained by\n        repeatedly calling ``run_model_on_task``.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n\n        trace_content : List[List]\n            Trace content obtained by ``openml.runs.run_flow_on_task``.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        if not hasattr(model, \"cv_results_\"):\n            raise ValueError(\"model should contain `cv_results_`\")\n\n        # attributes that will be in trace arff, regardless of the model\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n\n        # model dependent attributes for trace arff\n        for key in model.cv_results_:\n            if key.startswith(\"param_\"):\n                # supported types should include all types, including bool,\n                # int float\n                supported_basic_types = (bool, int, float, str)\n                for param_value in model.cv_results_[key]:\n                    if isinstance(param_value, np.generic):\n                        param_value = param_value.tolist()  # noqa: PLW2901\n                    if (\n                        isinstance(param_value, supported_basic_types)\n                        or param_value is None\n                        or param_value is np.ma.masked\n                    ):\n                        # basic string values\n                        type = \"STRING\"  # noqa: A001\n                    elif isinstance(param_value, (list, tuple)) and all(\n                        isinstance(i, int) for i in param_value\n                    ):\n                        # list of integers (usually for selecting features)\n                        # hyperparameter layer_sizes of MLPClassifier\n                        type = \"STRING\"  # noqa: A001\n                    else:\n                        raise TypeError(\"Unsupported param type in param grid: %s\" % key)\n\n                # renamed the attribute param to parameter, as this is a required\n                # OpenML convention - this also guards against name collisions\n                # with the required trace attributes\n                attribute = (PREFIX + key[6:], type)  # type: ignore\n                trace_attributes.append(attribute)\n\n        return OpenMLRunTrace.generate(\n            trace_attributes,\n            trace_content,\n        )\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>classmethod</code>","text":"<p>Check whether a given describes a scikit-learn estimator.</p> <p>This is done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given describes a scikit-learn estimator.\n\n    This is done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return cls._is_sklearn_flow(flow)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>classmethod</code>","text":"<p>Check whether a model is an instance of <code>sklearn.base.BaseEstimator</code>.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return isinstance(model, sklearn.base.BaseEstimator)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    from sklearn.exceptions import NotFittedError\n    from sklearn.utils.validation import check_is_fitted\n\n    try:\n        # check if model is fitted\n        check_is_fitted(model)\n\n        # Creating random dummy data of arbitrary size\n        dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n        # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n        # robust check that works across sklearn versions and models. Internally, 'predict'\n        # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n        # assured check than explicit calls to 'check_is_fitted'\n        model.predict(dummy_data)\n        # Will reach here if the model was fit on a dataset with 3 features\n        return True\n    except NotFittedError:  # needs to be the first exception to be caught\n        # Model is not fitted, as is required\n        return False\n    except ValueError:\n        # Will reach here if the model was fit on a dataset with more or less than 3 features\n        return True\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.create_setup_string","title":"<code>create_setup_string(model)</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n    return \" \".join(self.get_version_information())\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>","text":"<p>Initializes a sklearn model based on a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>mixed</code> <p>the object to deserialize (can be flow object, or any serialized parameter value that is accepted by)</p> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>mixed</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; Any:\n    \"\"\"Initializes a sklearn model based on a flow.\n\n    Parameters\n    ----------\n    flow : mixed\n        the object to deserialize (can be flow object, or any serialized\n        parameter value that is accepted by)\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    mixed\n    \"\"\"\n    return self._deserialize_sklearn(\n        flow,\n        initialize_with_defaults=initialize_with_defaults,\n        strict_version=strict_version,\n    )\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.get_version_information","title":"<code>get_version_information()</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Libraries listed are <code>Python</code>, <code>scikit-learn</code>, <code>numpy</code> and <code>scipy</code>.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n    Returns\n    -------\n    List\n    \"\"\"\n    # This can possibly be done by a package such as pyxb, but I could not get\n    # it to work properly.\n    import numpy\n    import scipy\n    import sklearn\n\n    major, minor, micro, _, _ = sys.version_info\n    python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n    sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n    numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n    scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n    return [python_version, sklearn_version, numpy_version, scipy_version]\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>","text":"<p>Instantiate a <code>base_estimator</code> which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n    optimization model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n    if not self._is_hpo_class(model):\n        raise AssertionError(\n            \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n            % model,\n        )\n    base_estimator = model.estimator\n    base_estimator.set_params(**trace_iteration.get_parameters())\n    return base_estimator\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.is_estimator","title":"<code>is_estimator(model)</code>","text":"<p>Check whether the given model is a scikit-learn estimator.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is a scikit-learn estimator.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    o = model\n    return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.model_to_flow","title":"<code>model_to_flow(model)</code>","text":"<p>Transform a scikit-learn model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n    # Necessary to make pypy not complain about all the different possible return types\n    return self._serialize_sklearn(model)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def obtain_parameter_values(  # noqa: C901, PLR0915\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n    openml.flows.functions._check_flow_for_server_id(flow)\n\n    def get_flow_dict(_flow):\n        flow_map = {_flow.name: _flow.flow_id}\n        for subflow in _flow.components:\n            flow_map.update(get_flow_dict(_flow.components[subflow]))\n        return flow_map\n\n    def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n        _flow,\n        _flow_dict,\n        component_model,\n        _main_call=False,  # noqa: FBT002\n        main_id=None,\n    ):\n        def is_subcomponent_specification(values):\n            # checks whether the current value can be a specification of\n            # subcomponents, as for example the value for steps parameter\n            # (in Pipeline) or transformers parameter (in\n            # ColumnTransformer).\n            return (\n                # Specification requires list/tuple of list/tuple with\n                # at least length 2.\n                isinstance(values, (tuple, list))\n                and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                # And each component needs to be a flow or interpretable string\n                and all(\n                    isinstance(item[1], openml.flows.OpenMLFlow)\n                    or (\n                        isinstance(item[1], str)\n                        and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    )\n                    for item in values\n                )\n            )\n\n        # _flow is openml flow object, _param dict maps from flow name to flow\n        # id for the main call, the param dict can be overridden (useful for\n        # unit tests / sentinels) this way, for flows without subflows we do\n        # not have to rely on _flow_dict\n        exp_parameters = set(_flow.parameters)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_parameters = set()\n        else:\n            model_parameters = set(component_model.get_params(deep=False))\n        if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n            flow_params = sorted(exp_parameters)\n            model_params = sorted(model_parameters)\n            raise ValueError(\n                \"Parameters of the model do not match the \"\n                \"parameters expected by the \"\n                \"flow:\\nexpected flow parameters: \"\n                f\"{flow_params}\\nmodel parameters: {model_params}\",\n            )\n        exp_components = set(_flow.components)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_components = set()\n        else:\n            _ = set(component_model.get_params(deep=False))\n            model_components = {\n                mp\n                for mp in component_model.get_params(deep=True)\n                if \"__\" not in mp and mp not in _\n            }\n        if len(exp_components.symmetric_difference(model_components)) != 0:\n            is_problem = True\n            if len(exp_components - model_components) &gt; 0:\n                # If an expected component is not returned as a component by get_params(),\n                # this means that it is also a parameter -&gt; we need to check that this is\n                # actually the case\n                difference = exp_components - model_components\n                component_in_model_parameters = []\n                for component in difference:\n                    if component in model_parameters:\n                        component_in_model_parameters.append(True)\n                    else:\n                        component_in_model_parameters.append(False)\n                is_problem = not all(component_in_model_parameters)\n            if is_problem:\n                flow_components = sorted(exp_components)\n                model_components = sorted(model_components)\n                raise ValueError(\n                    \"Subcomponents of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow subcomponents: \"\n                    f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                )\n\n        _params = []\n        for _param_name in _flow.parameters:\n            _current = OrderedDict()\n            _current[\"oml:name\"] = _param_name\n\n            current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n            # Try to filter out components (a.k.a. subflows) which are\n            # handled further down in the code (by recursively calling\n            # this function)!\n            if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                continue\n\n            if is_subcomponent_specification(current_param_values):\n                # complex parameter value, with subcomponents\n                parsed_values = []\n                for subcomponent in current_param_values:\n                    # scikit-learn stores usually tuples in the form\n                    # (name (str), subcomponent (mixed), argument\n                    # (mixed)). OpenML replaces the subcomponent by an\n                    # OpenMLFlow object.\n                    if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                        raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                    subcomponent_identifier = subcomponent[0]\n                    subcomponent_flow = subcomponent[1]\n                    if not isinstance(subcomponent_identifier, str):\n                        raise TypeError(\n                            \"Subcomponent identifier should be of type string, \"\n                            f\"but is {type(subcomponent_identifier)}\",\n                        )\n                    if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                        if (\n                            isinstance(subcomponent_flow, str)\n                            and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        ):\n                            pass\n                        else:\n                            raise TypeError(\n                                \"Subcomponent flow should be of type flow, but is {}\".format(\n                                    type(subcomponent_flow),\n                                ),\n                            )\n\n                    current = {\n                        \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                        \"value\": {\n                            \"key\": subcomponent_identifier,\n                            \"step_name\": subcomponent_identifier,\n                        },\n                    }\n                    if len(subcomponent) == 3:\n                        if not isinstance(subcomponent[2], list) and not isinstance(\n                            subcomponent[2],\n                            OrderedDict,\n                        ):\n                            raise TypeError(\n                                \"Subcomponent argument should be list or OrderedDict\",\n                            )\n                        current[\"value\"][\"argument_1\"] = subcomponent[2]\n                    parsed_values.append(current)\n                parsed_values = json.dumps(parsed_values)\n            else:\n                # vanilla parameter value\n                parsed_values = json.dumps(current_param_values)\n\n            _current[\"oml:value\"] = parsed_values\n            if _main_call:\n                _current[\"oml:component\"] = main_id\n            else:\n                _current[\"oml:component\"] = _flow_dict[_flow.name]\n            _params.append(_current)\n\n        for _identifier in _flow.components:\n            subcomponent_model = component_model.get_params()[_identifier]\n            _params.extend(\n                extract_parameters(\n                    _flow.components[_identifier],\n                    _flow_dict,\n                    subcomponent_model,\n                ),\n            )\n        return _params\n\n    flow_dict = get_flow_dict(flow)\n    model = model if model is not None else flow.model\n    return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.seed_model","title":"<code>seed_model(model, seed=None)</code>","text":"<p>Set the random state of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is raised when a RandomState was used as seed).</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> <p>The seed to initialize the RandomState with. Unseeded subcomponents will be seeded with a random number from the RandomState.</p> <code>None</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n    \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n    model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Models that are already seeded will maintain the seed. In this case,\n    only integer seeds are allowed (An exception is raised when a RandomState was used as\n    seed).\n\n    Parameters\n    ----------\n    model : sklearn model\n        The model to be seeded\n    seed : int\n        The seed to initialize the RandomState with. Unseeded subcomponents\n        will be seeded with a random number from the RandomState.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    def _seed_current_object(current_value):\n        if isinstance(current_value, int):  # acceptable behaviour\n            return False\n\n        if isinstance(current_value, np.random.RandomState):\n            raise ValueError(\n                \"Models initialized with a RandomState object are not \"\n                \"supported. Please seed with an integer. \",\n            )\n\n        if current_value is not None:\n            raise ValueError(\n                \"Models should be seeded with int or None (this should never \" \"happen). \",\n            )\n\n        return True\n\n    rs = np.random.RandomState(seed)\n    model_params = model.get_params()\n    random_states = {}\n    for param_name in sorted(model_params):\n        if \"random_state\" in param_name:\n            current_value = model_params[param_name]\n            # important to draw the value at this point (and not in the if\n            # statement) this way we guarantee that if a different set of\n            # subflows is seeded, the same number of the random generator is\n            # used\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                random_states[param_name] = new_value\n\n        # Also seed CV objects!\n        elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n            if not hasattr(model_params[param_name], \"random_state\"):\n                continue\n\n            current_value = model_params[param_name].random_state\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                model_params[param_name].random_state = new_value\n\n    model.set_params(**random_states)\n    return model\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.trim_flow_name","title":"<code>trim_flow_name(long_name, extra_trim_length=100, _outer=True)</code>  <code>classmethod</code>","text":"<p>Shorten generated sklearn flow name to at most <code>max_length</code> characters.</p> <p>Flows are assumed to have the following naming structure: <code>(model_selection)? (pipeline)? (steps)+</code> and will be shortened to: <code>sklearn.(selection.)?(pipeline.)?(steps)+</code> e.g. (white spaces and newlines added for readability)</p> <p>.. code ::</p> <pre><code>sklearn.pipeline.Pipeline(\n    columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n        numeric=sklearn.pipeline.Pipeline(\n            imputer=sklearn.preprocessing.imputation.Imputer,\n            standardscaler=sklearn.preprocessing.data.StandardScaler),\n        nominal=sklearn.pipeline.Pipeline(\n            simpleimputer=sklearn.impute.SimpleImputer,\n            onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n    variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n    svc=sklearn.svm.classes.SVC)\n</code></pre> <p>-&gt; <code>sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)</code></p> <p>Parameters:</p> Name Type Description Default <code>long_name</code> <code>str</code> <p>The full flow name generated by the scikit-learn extension.</p> required <code>extra_trim_length</code> <code>int</code> <p>If the trimmed name would exceed <code>extra_trim_length</code> characters, additional trimming of the short name is performed. This reduces the produced short name length. There is no guarantee the end result will not exceed <code>extra_trim_length</code>.</p> <code>100</code> <code>_outer</code> <code>bool(default=True)</code> <p>For internal use only. Specifies if the function is called recursively.</p> <code>True</code> <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef trim_flow_name(  # noqa: C901\n    cls,\n    long_name: str,\n    extra_trim_length: int = 100,\n    _outer: bool = True,  # noqa: FBT001, FBT002\n) -&gt; str:\n    \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n    Flows are assumed to have the following naming structure:\n    ``(model_selection)? (pipeline)? (steps)+``\n    and will be shortened to:\n    ``sklearn.(selection.)?(pipeline.)?(steps)+``\n    e.g. (white spaces and newlines added for readability)\n\n    .. code ::\n\n        sklearn.pipeline.Pipeline(\n            columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                numeric=sklearn.pipeline.Pipeline(\n                    imputer=sklearn.preprocessing.imputation.Imputer,\n                    standardscaler=sklearn.preprocessing.data.StandardScaler),\n                nominal=sklearn.pipeline.Pipeline(\n                    simpleimputer=sklearn.impute.SimpleImputer,\n                    onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n            variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n            svc=sklearn.svm.classes.SVC)\n\n    -&gt;\n    ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n    Parameters\n    ----------\n    long_name : str\n        The full flow name generated by the scikit-learn extension.\n    extra_trim_length: int (default=100)\n        If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n        of the short name is performed. This reduces the produced short name length.\n        There is no guarantee the end result will not exceed `extra_trim_length`.\n    _outer : bool (default=True)\n        For internal use only. Specifies if the function is called recursively.\n\n    Returns\n    -------\n    str\n\n    \"\"\"\n\n    def remove_all_in_parentheses(string: str) -&gt; str:\n        string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        while removals &gt; 0:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        return string\n\n    # Generally, we want to trim all hyperparameters, the exception to that is for model\n    # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n    # So we first trim name of the `estimator` specified in mode selection. For reference, in\n    # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n    # keep it in the final trimmed flow name:\n    # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n    # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n    # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n    # sklearn.tree.tree.DecisionTreeClassifier))\n    if \"sklearn.model_selection\" in long_name:\n        start_index = long_name.index(\"sklearn.model_selection\")\n        estimator_start = (\n            start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n        )\n\n        model_select_boilerplate = long_name[start_index:estimator_start]\n        # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n        model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n        # Now we want to also find and parse the `estimator`, for this we find the closing\n        # parenthesis to the model selection technique:\n        closing_parenthesis_expected = 1\n        for char in long_name[estimator_start:]:\n            if char == \"(\":\n                closing_parenthesis_expected += 1\n            if char == \")\":\n                closing_parenthesis_expected -= 1\n            if closing_parenthesis_expected == 0:\n                break\n\n        _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n        model_select_pipeline = long_name[estimator_start:_end]\n\n        trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n        _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n        model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n        name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n    else:\n        name = long_name\n\n    module_name = long_name.split(\".\")[0]\n    short_name = module_name + \".{}\"\n\n    if name.startswith(\"sklearn.pipeline\"):\n        full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n        pipeline_class = full_pipeline_class.split(\".\")[-1]\n        # We don't want nested pipelines in the short name, so we trim all complicated\n        # subcomponents, i.e. those with parentheses:\n        pipeline = remove_all_in_parentheses(pipeline)\n\n        # then the pipeline steps are formatted e.g.:\n        # step1name=sklearn.submodule.ClassName,step2name...\n        components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n        pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n        if len(short_name.format(pipeline)) &gt; extra_trim_length:\n            pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n    else:\n        # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n        pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n    if not _outer:\n        # Anything from parenthesis in inner calls should not be culled, so we use brackets\n        pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n    else:\n        # Square brackets may be introduced with nested model_selection\n        pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n    return short_name.format(pipeline)\n</code></pre>"},{"location":"reference/flows/","title":"flows","text":""},{"location":"reference/flows/#openml.flows.OpenMLFlow","title":"<code>OpenMLFlow</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Flow. Stores machine learning models.</p> <p>Flows should not be generated manually, but by the function :meth:<code>openml.flows.create_flow_from_model</code>. Using this helper function ensures that all relevant fields are filled in.</p> <p>Implements <code>openml.implementation.upload.xsd &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/ openml.implementation.upload.xsd&gt;</code>_.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the flow. Is used together with the attribute <code>external_version</code> as a unique identifier of the flow.</p> required <code>description</code> <code>str</code> <p>Human-readable description of the flow (free text).</p> required <code>model</code> <code>object</code> <p>ML model which is described by this flow.</p> required <code>components</code> <code>OrderedDict</code> <p>Mapping from component identifier to an OpenMLFlow object. Components are usually subfunctions of an algorithm (e.g. kernels), base learners in ensemble algorithms (decision tree in adaboost) or building blocks of a machine learning pipeline. Components are modeled as independent flows and can be shared between flows (different pipelines can use the same components).</p> required <code>parameters</code> <code>OrderedDict</code> <p>Mapping from parameter name to the parameter default value. The parameter default value must be of type <code>str</code>, so that the respective toolbox plugin can take care of casting the parameter default value to the correct type.</p> required <code>parameters_meta_info</code> <code>OrderedDict</code> <p>Mapping from parameter name to <code>dict</code>. Stores additional information for each parameter. Required keys are <code>data_type</code> and <code>description</code>.</p> required <code>external_version</code> <code>str</code> <p>Version number of the software the flow is implemented in. Is used together with the attribute <code>name</code> as a uniquer identifier of the flow.</p> required <code>tags</code> <code>list</code> <p>List of tags. Created on the server by other API calls.</p> required <code>language</code> <code>str</code> <p>Natural language the flow is described in (not the programming language).</p> required <code>dependencies</code> <code>str</code> <p>A list of dependencies necessary to run the flow. This field should contain all libraries the flow depends on. To allow reproducibility it should also specify the exact version numbers.</p> required <code>class_name</code> <code>str</code> <p>The development language name of the class which is described by this flow.</p> <code>None</code> <code>custom_name</code> <code>str</code> <p>Custom name of the flow given by the owner.</p> <code>None</code> <code>binary_url</code> <code>str</code> <p>Url from which the binary can be downloaded. Added by the server. Ignored when uploaded manually. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_format</code> <code>str</code> <p>Format in which the binary code was uploaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_md5</code> <code>str</code> <p>MD5 checksum to check if the binary code was correctly downloaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>uploader</code> <code>str</code> <p>OpenML user ID of the uploader. Filled in by the server.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>Date the flow was uploaded. Filled in by the server.</p> <code>None</code> <code>flow_id</code> <code>int</code> <p>Flow ID. Assigned by the server.</p> <code>None</code> <code>extension</code> <code>Extension</code> <p>The extension for a flow (e.g., sklearn).</p> <code>None</code> <code>version</code> <code>str</code> <p>OpenML version of the flow. Assigned by the server.</p> <code>None</code> Source code in <code>openml/flows/flow.py</code> <pre><code>class OpenMLFlow(OpenMLBase):\n    \"\"\"OpenML Flow. Stores machine learning models.\n\n    Flows should not be generated manually, but by the function\n    :meth:`openml.flows.create_flow_from_model`. Using this helper function\n    ensures that all relevant fields are filled in.\n\n    Implements `openml.implementation.upload.xsd\n    &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/\n    openml.implementation.upload.xsd&gt;`_.\n\n    Parameters\n    ----------\n    name : str\n        Name of the flow. Is used together with the attribute\n        `external_version` as a unique identifier of the flow.\n    description : str\n        Human-readable description of the flow (free text).\n    model : object\n        ML model which is described by this flow.\n    components : OrderedDict\n        Mapping from component identifier to an OpenMLFlow object. Components\n        are usually subfunctions of an algorithm (e.g. kernels), base learners\n        in ensemble algorithms (decision tree in adaboost) or building blocks\n        of a machine learning pipeline. Components are modeled as independent\n        flows and can be shared between flows (different pipelines can use\n        the same components).\n    parameters : OrderedDict\n        Mapping from parameter name to the parameter default value. The\n        parameter default value must be of type `str`, so that the respective\n        toolbox plugin can take care of casting the parameter default value to\n        the correct type.\n    parameters_meta_info : OrderedDict\n        Mapping from parameter name to `dict`. Stores additional information\n        for each parameter. Required keys are `data_type` and `description`.\n    external_version : str\n        Version number of the software the flow is implemented in. Is used\n        together with the attribute `name` as a uniquer identifier of the flow.\n    tags : list\n        List of tags. Created on the server by other API calls.\n    language : str\n        Natural language the flow is described in (not the programming\n        language).\n    dependencies : str\n        A list of dependencies necessary to run the flow. This field should\n        contain all libraries the flow depends on. To allow reproducibility\n        it should also specify the exact version numbers.\n    class_name : str, optional\n        The development language name of the class which is described by this\n        flow.\n    custom_name : str, optional\n        Custom name of the flow given by the owner.\n    binary_url : str, optional\n        Url from which the binary can be downloaded. Added by the server.\n        Ignored when uploaded manually. Will not be used by the python API\n        because binaries aren't compatible across machines.\n    binary_format : str, optional\n        Format in which the binary code was uploaded. Will not be used by the\n        python API because binaries aren't compatible across machines.\n    binary_md5 : str, optional\n        MD5 checksum to check if the binary code was correctly downloaded. Will\n        not be used by the python API because binaries aren't compatible across\n        machines.\n    uploader : str, optional\n        OpenML user ID of the uploader. Filled in by the server.\n    upload_date : str, optional\n        Date the flow was uploaded. Filled in by the server.\n    flow_id : int, optional\n        Flow ID. Assigned by the server.\n    extension : Extension, optional\n        The extension for a flow (e.g., sklearn).\n    version : str, optional\n        OpenML version of the flow. Assigned by the server.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        name: str,\n        description: str,\n        model: object,\n        components: dict,\n        parameters: dict,\n        parameters_meta_info: dict,\n        external_version: str,\n        tags: list,\n        language: str,\n        dependencies: str,\n        class_name: str | None = None,\n        custom_name: str | None = None,\n        binary_url: str | None = None,\n        binary_format: str | None = None,\n        binary_md5: str | None = None,\n        uploader: str | None = None,\n        upload_date: str | None = None,\n        flow_id: int | None = None,\n        extension: Extension | None = None,\n        version: str | None = None,\n    ):\n        self.name = name\n        self.description = description\n        self.model = model\n\n        for variable, variable_name in [\n            [components, \"components\"],\n            [parameters, \"parameters\"],\n            [parameters_meta_info, \"parameters_meta_info\"],\n        ]:\n            if not isinstance(variable, (OrderedDict, dict)):\n                raise TypeError(\n                    f\"{variable_name} must be of type OrderedDict or dict, \"\n                    f\"but is {type(variable)}.\",\n                )\n\n        self.components = components\n        self.parameters = parameters\n        self.parameters_meta_info = parameters_meta_info\n        self.class_name = class_name\n\n        keys_parameters = set(parameters.keys())\n        keys_parameters_meta_info = set(parameters_meta_info.keys())\n        if len(keys_parameters.difference(keys_parameters_meta_info)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters, but not in \"\n                \"parameters_meta_info.\"\n                % str(keys_parameters.difference(keys_parameters_meta_info)),\n            )\n        if len(keys_parameters_meta_info.difference(keys_parameters)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters_meta_info, \"\n                \"but not in parameters.\"\n                % str(keys_parameters_meta_info.difference(keys_parameters)),\n            )\n\n        self.external_version = external_version\n        self.uploader = uploader\n\n        self.custom_name = custom_name\n        self.tags = tags if tags is not None else []\n        self.binary_url = binary_url\n        self.binary_format = binary_format\n        self.binary_md5 = binary_md5\n        self.version = version\n        self.upload_date = upload_date\n        self.language = language\n        self.dependencies = dependencies\n        self.flow_id = flow_id\n        if extension is None:\n            self._extension = get_extension_by_flow(self)\n        else:\n            self._extension = extension\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the flow.\"\"\"\n        return self.flow_id\n\n    @property\n    def extension(self) -&gt; Extension:\n        \"\"\"The extension of the flow (e.g., sklearn).\"\"\"\n        if self._extension is not None:\n            return self._extension\n\n        raise RuntimeError(\n            f\"No extension could be found for flow {self.flow_id}: {self.name}\",\n        )\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields = {\n            \"Flow Name\": self.name,\n            \"Flow Description\": self.description,\n            \"Dependencies\": self.dependencies,\n        }\n        if self.flow_id is not None:\n            fields[\"Flow URL\"] = self.openml_url if self.openml_url is not None else \"None\"\n            fields[\"Flow ID\"] = str(self.flow_id)\n            if self.version is not None:\n                fields[\"Flow ID\"] += f\" (version {self.version})\"\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.binary_url is not None:\n            fields[\"Binary URL\"] = self.binary_url\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Flow ID\",\n            \"Flow URL\",\n            \"Flow Name\",\n            \"Flow Description\",\n            \"Binary URL\",\n            \"Upload Date\",\n            \"Dependencies\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: C901, PLR0912\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        flow_container = OrderedDict()  # type: 'dict[str, dict]'\n        flow_dict = OrderedDict(\n            [(\"@xmlns:oml\", \"http://openml.org/openml\")],\n        )  # type: 'dict[str, list | str]'  # E501\n        flow_container[\"oml:flow\"] = flow_dict\n        _add_if_nonempty(flow_dict, \"oml:id\", self.flow_id)\n\n        for required in [\"name\", \"external_version\"]:\n            if getattr(self, required) is None:\n                raise ValueError(f\"self.{required} is required but None\")\n        for attribute in [\n            \"uploader\",\n            \"name\",\n            \"custom_name\",\n            \"class_name\",\n            \"version\",\n            \"external_version\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n        ]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        if not self.description:\n            logger = logging.getLogger(__name__)\n            logger.warning(\"Flow % has empty description\", self.name)\n\n        flow_parameters = []\n        for key in self.parameters:\n            param_dict = OrderedDict()  # type: 'OrderedDict[str, str]'\n            param_dict[\"oml:name\"] = key\n            meta_info = self.parameters_meta_info[key]\n\n            _add_if_nonempty(param_dict, \"oml:data_type\", meta_info[\"data_type\"])\n            param_dict[\"oml:default_value\"] = self.parameters[key]\n            _add_if_nonempty(param_dict, \"oml:description\", meta_info[\"description\"])\n\n            for key_, value in param_dict.items():\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n                if value is not None and not isinstance(value, str):\n                    raise ValueError(\n                        f\"Parameter value {value} cannot be serialized \"\n                        f\"because it is of type {type(value)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            flow_parameters.append(param_dict)\n\n        flow_dict[\"oml:parameter\"] = flow_parameters\n\n        components = []\n        for key in self.components:\n            component_dict = OrderedDict()  # type: 'OrderedDict[str, dict]'\n            component_dict[\"oml:identifier\"] = key\n            if self.components[key] in [\"passthrough\", \"drop\"]:\n                component_dict[\"oml:flow\"] = {\n                    \"oml-python:serialized_object\": \"component_reference\",\n                    \"value\": {\"key\": self.components[key], \"step_name\": self.components[key]},\n                }\n            else:\n                component_dict[\"oml:flow\"] = self.components[key]._to_dict()[\"oml:flow\"]\n\n            for key_ in component_dict:\n                # We only need to check if the key is a string, because the\n                # value is a flow. The flow itself is valid by recursion\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            components.append(component_dict)\n\n        flow_dict[\"oml:component\"] = components\n        flow_dict[\"oml:tag\"] = self.tags\n        for attribute in [\"binary_url\", \"binary_format\", \"binary_md5\"]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        return flow_container\n\n    @classmethod\n    def _from_dict(cls, xml_dict: dict) -&gt; OpenMLFlow:\n        \"\"\"Create a flow from an xml description.\n\n        Calls itself recursively to create :class:`OpenMLFlow` objects of\n        subflows (components).\n\n        XML definition of a flow is available at\n        https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd\n\n        Parameters\n        ----------\n        xml_dict : dict\n            Dictionary representation of the flow as created by _to_dict()\n\n        Returns\n        -------\n            OpenMLFlow\n\n        \"\"\"  # E501\n        arguments = OrderedDict()\n        dic = xml_dict[\"oml:flow\"]\n\n        # Mandatory parts in the xml file\n        for key in [\"name\"]:\n            arguments[key] = dic[\"oml:\" + key]\n\n        # non-mandatory parts in the xml file\n        for key in [\n            \"external_version\",\n            \"uploader\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n            \"version\",\n            \"binary_url\",\n            \"binary_format\",\n            \"binary_md5\",\n            \"class_name\",\n            \"custom_name\",\n        ]:\n            arguments[key] = dic.get(\"oml:\" + key)\n\n        # has to be converted to an int if present and cannot parsed in the\n        # two loops above\n        arguments[\"flow_id\"] = int(dic[\"oml:id\"]) if dic.get(\"oml:id\") is not None else None\n\n        # Now parse parts of a flow which can occur multiple times like\n        # parameters, components (subflows) and tags. These can't be tackled\n        # in the loops above because xmltodict returns a dict if such an\n        # entity occurs once, and a list if it occurs multiple times.\n        # Furthermore, they must be treated differently, for example\n        # for components this method is called recursively and\n        # for parameters the actual information is split into two dictionaries\n        # for easier access in python.\n\n        parameters = OrderedDict()\n        parameters_meta_info = OrderedDict()\n        if \"oml:parameter\" in dic:\n            # In case of a single parameter, xmltodict returns a dictionary,\n            # otherwise a list.\n            oml_parameters = extract_xml_tags(\"oml:parameter\", dic, allow_none=False)\n\n            for oml_parameter in oml_parameters:\n                parameter_name = oml_parameter[\"oml:name\"]\n                default_value = oml_parameter[\"oml:default_value\"]\n                parameters[parameter_name] = default_value\n\n                meta_info = OrderedDict()\n                meta_info[\"description\"] = oml_parameter.get(\"oml:description\")\n                meta_info[\"data_type\"] = oml_parameter.get(\"oml:data_type\")\n                parameters_meta_info[parameter_name] = meta_info\n        arguments[\"parameters\"] = parameters\n        arguments[\"parameters_meta_info\"] = parameters_meta_info\n\n        components = OrderedDict()\n        if \"oml:component\" in dic:\n            # In case of a single component xmltodict returns a dict,\n            # otherwise a list.\n            oml_components = extract_xml_tags(\"oml:component\", dic, allow_none=False)\n\n            for component in oml_components:\n                flow = OpenMLFlow._from_dict(component)\n                components[component[\"oml:identifier\"]] = flow\n        arguments[\"components\"] = components\n        arguments[\"tags\"] = extract_xml_tags(\"oml:tag\", dic)\n\n        arguments[\"model\"] = None\n        return cls(**arguments)\n\n    def to_filesystem(self, output_directory: str | Path) -&gt; None:\n        \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n        output_directory = Path(output_directory)\n        output_directory.mkdir(parents=True, exist_ok=True)\n\n        output_path = output_directory / \"flow.xml\"\n        if output_path.exists():\n            raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n        run_xml = self._to_xml()\n        with output_path.open(\"w\") as f:\n            f.write(run_xml)\n\n    @classmethod\n    def from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n        \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n        input_directory = Path(input_directory) / \"flow.xml\"\n        with input_directory.open() as f:\n            xml_string = f.read()\n        return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.flow_id = int(xml_response[\"oml:upload_flow\"][\"oml:id\"])\n\n    def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n        \"\"\"Publish this flow to OpenML server.\n\n        Raises a PyOpenMLError if the flow exists on the server, but\n        `self.flow_id` does not match the server known flow id.\n\n        Parameters\n        ----------\n        raise_error_if_exists : bool, optional (default=False)\n            If True, raise PyOpenMLError if the flow exists on the server.\n            If False, update the local flow to match the server flow.\n\n        Returns\n        -------\n        self : OpenMLFlow\n\n        \"\"\"\n        # Import at top not possible because of cyclic dependencies. In\n        # particular, flow.py tries to import functions.py in order to call\n        # get_flow(), while functions.py tries to import flow.py in order to\n        # instantiate an OpenMLFlow.\n        import openml.flows.functions\n\n        flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n        if not flow_id:\n            if self.flow_id:\n                raise openml.exceptions.PyOpenMLError(\n                    \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n                )\n            super().publish()\n            assert self.flow_id is not None  # for mypy\n            flow_id = self.flow_id\n        elif raise_error_if_exists:\n            error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n            raise openml.exceptions.PyOpenMLError(error_message)\n        elif self.flow_id is not None and self.flow_id != flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n            )\n\n        flow = openml.flows.functions.get_flow(flow_id)\n        _copy_server_fields(flow, self)\n        try:\n            openml.flows.functions.assert_flows_equal(\n                self,\n                flow,\n                flow.upload_date,\n                ignore_parameter_values=True,\n                ignore_custom_name_if_none=True,\n            )\n        except ValueError as e:\n            message = e.args[0]\n            raise ValueError(\n                \"The flow on the server is inconsistent with the local flow. \"\n                f\"The server flow ID is {flow_id}. Please check manually and remove \"\n                f\"the flow if necessary! Error is:\\n'{message}'\",\n            ) from e\n        return self\n\n    def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n        \"\"\"\n        Returns for each sub-component of the flow the path of identifiers\n        that should be traversed to reach this component. The resulting dict\n        maps a key (identifying a flow by either its id, name or fullname) to\n        the parameter prefix.\n\n        Parameters\n        ----------\n        key_item: str\n            The flow attribute that will be used to identify flows in the\n            structure. Allowed values {flow_id, name}\n\n        Returns\n        -------\n        dict[str, List[str]]\n            The flow structure\n        \"\"\"\n        if key_item not in [\"flow_id\", \"name\"]:\n            raise ValueError(\"key_item should be in {flow_id, name}\")\n        structure = {}\n        for key, sub_flow in self.components.items():\n            sub_structure = sub_flow.get_structure(key_item)\n            for flow_name, flow_sub_structure in sub_structure.items():\n                structure[flow_name] = [key, *flow_sub_structure]\n        structure[getattr(self, key_item)] = []\n        return structure\n\n    def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n        \"\"\"\n        Returns a subflow from the tree of dependencies.\n\n        Parameters\n        ----------\n        structure: list[str]\n            A list of strings, indicating the location of the subflow\n\n        Returns\n        -------\n        OpenMLFlow\n            The OpenMLFlow that corresponds to the structure\n        \"\"\"\n        # make a copy of structure, as we don't want to change it in the\n        # outer scope\n        structure = list(structure)\n        if len(structure) &lt; 1:\n            raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n        sub_identifier = structure[0]\n        if sub_identifier not in self.components:\n            raise ValueError(\n                f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n            )\n        if len(structure) == 1:\n            return self.components[sub_identifier]  # type: ignore\n\n        structure.pop(0)\n        return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.extension","title":"<code>extension: Extension</code>  <code>property</code>","text":"<p>The extension of the flow (e.g., sklearn).</p>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the flow.</p>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.from_filesystem","title":"<code>from_filesystem(input_directory)</code>  <code>classmethod</code>","text":"<p>Read a flow from an XML in input_directory on the filesystem.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n    \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n    input_directory = Path(input_directory) / \"flow.xml\"\n    with input_directory.open() as f:\n        xml_string = f.read()\n    return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.get_structure","title":"<code>get_structure(key_item)</code>","text":"<p>Returns for each sub-component of the flow the path of identifiers that should be traversed to reach this component. The resulting dict maps a key (identifying a flow by either its id, name or fullname) to the parameter prefix.</p> <p>Parameters:</p> Name Type Description Default <code>key_item</code> <code>str</code> <p>The flow attribute that will be used to identify flows in the structure. Allowed values {flow_id, name}</p> required <p>Returns:</p> Type Description <code>dict[str, List[str]]</code> <p>The flow structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n    \"\"\"\n    Returns for each sub-component of the flow the path of identifiers\n    that should be traversed to reach this component. The resulting dict\n    maps a key (identifying a flow by either its id, name or fullname) to\n    the parameter prefix.\n\n    Parameters\n    ----------\n    key_item: str\n        The flow attribute that will be used to identify flows in the\n        structure. Allowed values {flow_id, name}\n\n    Returns\n    -------\n    dict[str, List[str]]\n        The flow structure\n    \"\"\"\n    if key_item not in [\"flow_id\", \"name\"]:\n        raise ValueError(\"key_item should be in {flow_id, name}\")\n    structure = {}\n    for key, sub_flow in self.components.items():\n        sub_structure = sub_flow.get_structure(key_item)\n        for flow_name, flow_sub_structure in sub_structure.items():\n            structure[flow_name] = [key, *flow_sub_structure]\n    structure[getattr(self, key_item)] = []\n    return structure\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.get_subflow","title":"<code>get_subflow(structure)</code>","text":"<p>Returns a subflow from the tree of dependencies.</p> <p>Parameters:</p> Name Type Description Default <code>structure</code> <code>list[str]</code> <p>A list of strings, indicating the location of the subflow</p> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> <p>The OpenMLFlow that corresponds to the structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n    \"\"\"\n    Returns a subflow from the tree of dependencies.\n\n    Parameters\n    ----------\n    structure: list[str]\n        A list of strings, indicating the location of the subflow\n\n    Returns\n    -------\n    OpenMLFlow\n        The OpenMLFlow that corresponds to the structure\n    \"\"\"\n    # make a copy of structure, as we don't want to change it in the\n    # outer scope\n    structure = list(structure)\n    if len(structure) &lt; 1:\n        raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n    sub_identifier = structure[0]\n    if sub_identifier not in self.components:\n        raise ValueError(\n            f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n        )\n    if len(structure) == 1:\n        return self.components[sub_identifier]  # type: ignore\n\n    structure.pop(0)\n    return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.publish","title":"<code>publish(raise_error_if_exists=False)</code>","text":"<p>Publish this flow to OpenML server.</p> <p>Raises a PyOpenMLError if the flow exists on the server, but <code>self.flow_id</code> does not match the server known flow id.</p> <p>Parameters:</p> Name Type Description Default <code>raise_error_if_exists</code> <code>(bool, optional(default=False))</code> <p>If True, raise PyOpenMLError if the flow exists on the server. If False, update the local flow to match the server flow.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>self</code> <code>OpenMLFlow</code> Source code in <code>openml/flows/flow.py</code> <pre><code>def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Publish this flow to OpenML server.\n\n    Raises a PyOpenMLError if the flow exists on the server, but\n    `self.flow_id` does not match the server known flow id.\n\n    Parameters\n    ----------\n    raise_error_if_exists : bool, optional (default=False)\n        If True, raise PyOpenMLError if the flow exists on the server.\n        If False, update the local flow to match the server flow.\n\n    Returns\n    -------\n    self : OpenMLFlow\n\n    \"\"\"\n    # Import at top not possible because of cyclic dependencies. In\n    # particular, flow.py tries to import functions.py in order to call\n    # get_flow(), while functions.py tries to import flow.py in order to\n    # instantiate an OpenMLFlow.\n    import openml.flows.functions\n\n    flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n    if not flow_id:\n        if self.flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n            )\n        super().publish()\n        assert self.flow_id is not None  # for mypy\n        flow_id = self.flow_id\n    elif raise_error_if_exists:\n        error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n        raise openml.exceptions.PyOpenMLError(error_message)\n    elif self.flow_id is not None and self.flow_id != flow_id:\n        raise openml.exceptions.PyOpenMLError(\n            \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n        )\n\n    flow = openml.flows.functions.get_flow(flow_id)\n    _copy_server_fields(flow, self)\n    try:\n        openml.flows.functions.assert_flows_equal(\n            self,\n            flow,\n            flow.upload_date,\n            ignore_parameter_values=True,\n            ignore_custom_name_if_none=True,\n        )\n    except ValueError as e:\n        message = e.args[0]\n        raise ValueError(\n            \"The flow on the server is inconsistent with the local flow. \"\n            f\"The server flow ID is {flow_id}. Please check manually and remove \"\n            f\"the flow if necessary! Error is:\\n'{message}'\",\n        ) from e\n    return self\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.to_filesystem","title":"<code>to_filesystem(output_directory)</code>","text":"<p>Write a flow to the filesystem as XML to output_directory.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def to_filesystem(self, output_directory: str | Path) -&gt; None:\n    \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n    output_directory = Path(output_directory)\n    output_directory.mkdir(parents=True, exist_ok=True)\n\n    output_path = output_directory / \"flow.xml\"\n    if output_path.exists():\n        raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n    run_xml = self._to_xml()\n    with output_path.open(\"w\") as f:\n        f.write(run_xml)\n</code></pre>"},{"location":"reference/flows/#openml.flows.assert_flows_equal","title":"<code>assert_flows_equal(flow1, flow2, ignore_parameter_values_on_older_children=None, ignore_parameter_values=False, ignore_custom_name_if_none=False, check_description=True)</code>","text":"<p>Check equality of two flows.</p> <p>Two flows are equal if their all keys which are not set by the server are equal, as well as all their parameters and components.</p> <p>Parameters:</p> Name Type Description Default <code>flow1</code> <code>OpenMLFlow</code> required <code>flow2</code> <code>OpenMLFlow</code> required <code>ignore_parameter_values_on_older_children</code> <code>str(optional)</code> <p>If set to <code>OpenMLFlow.upload_date</code>, ignores parameters in a child flow if it's upload date predates the upload date of the parent flow.</p> <code>None</code> <code>ignore_parameter_values</code> <code>bool</code> <p>Whether to ignore parameter values when comparing flows.</p> <code>False</code> <code>ignore_custom_name_if_none</code> <code>bool</code> <p>Whether to ignore the custom name field if either flow has <code>custom_name</code> equal to <code>None</code>.</p> <code>False</code> <code>check_description</code> <code>bool</code> <p>Whether to ignore matching of flow descriptions.</p> <code>True</code> Source code in <code>openml/flows/functions.py</code> <pre><code>def assert_flows_equal(  # noqa: C901, PLR0912, PLR0913, PLR0915\n    flow1: OpenMLFlow,\n    flow2: OpenMLFlow,\n    ignore_parameter_values_on_older_children: str | None = None,\n    ignore_parameter_values: bool = False,  # noqa: FBT001, FBT002\n    ignore_custom_name_if_none: bool = False,  # noqa:  FBT001, FBT002\n    check_description: bool = True,  # noqa:  FBT001, FBT002\n) -&gt; None:\n    \"\"\"Check equality of two flows.\n\n    Two flows are equal if their all keys which are not set by the server\n    are equal, as well as all their parameters and components.\n\n    Parameters\n    ----------\n    flow1 : OpenMLFlow\n\n    flow2 : OpenMLFlow\n\n    ignore_parameter_values_on_older_children : str (optional)\n        If set to ``OpenMLFlow.upload_date``, ignores parameters in a child\n        flow if it's upload date predates the upload date of the parent flow.\n\n    ignore_parameter_values : bool\n        Whether to ignore parameter values when comparing flows.\n\n    ignore_custom_name_if_none : bool\n        Whether to ignore the custom name field if either flow has `custom_name` equal to `None`.\n\n    check_description : bool\n        Whether to ignore matching of flow descriptions.\n    \"\"\"\n    if not isinstance(flow1, OpenMLFlow):\n        raise TypeError(\"Argument 1 must be of type OpenMLFlow, but is %s\" % type(flow1))\n\n    if not isinstance(flow2, OpenMLFlow):\n        raise TypeError(\"Argument 2 must be of type OpenMLFlow, but is %s\" % type(flow2))\n\n    # TODO as they are actually now saved during publish, it might be good to\n    # check for the equality of these as well.\n    generated_by_the_server = [\n        \"flow_id\",\n        \"uploader\",\n        \"version\",\n        \"upload_date\",\n        # Tags aren't directly created by the server,\n        # but the uploader has no control over them!\n        \"tags\",\n    ]\n    ignored_by_python_api = [\"binary_url\", \"binary_format\", \"binary_md5\", \"model\", \"_entity_id\"]\n\n    for key in set(flow1.__dict__.keys()).union(flow2.__dict__.keys()):\n        if key in generated_by_the_server + ignored_by_python_api:\n            continue\n        attr1 = getattr(flow1, key, None)\n        attr2 = getattr(flow2, key, None)\n        if key == \"components\":\n            if not (isinstance(attr1, Dict) and isinstance(attr2, Dict)):\n                raise TypeError(\"Cannot compare components because they are not dictionary.\")\n\n            for name in set(attr1.keys()).union(attr2.keys()):\n                if name not in attr1:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                if name not in attr2:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                assert_flows_equal(\n                    attr1[name],\n                    attr2[name],\n                    ignore_parameter_values_on_older_children,\n                    ignore_parameter_values,\n                    ignore_custom_name_if_none,\n                )\n        elif key == \"_extension\":\n            continue\n        elif check_description and key == \"description\":\n            # to ignore matching of descriptions since sklearn based flows may have\n            # altering docstrings and is not guaranteed to be consistent\n            continue\n        else:\n            if key == \"parameters\":\n                if ignore_parameter_values or ignore_parameter_values_on_older_children:\n                    params_flow_1 = set(flow1.parameters.keys())\n                    params_flow_2 = set(flow2.parameters.keys())\n                    symmetric_difference = params_flow_1 ^ params_flow_2\n                    if len(symmetric_difference) &gt; 0:\n                        raise ValueError(\n                            \"Flow %s: parameter set of flow \"\n                            \"differs from the parameters stored \"\n                            \"on the server.\" % flow1.name,\n                        )\n\n                if ignore_parameter_values_on_older_children:\n                    assert (\n                        flow1.upload_date is not None\n                    ), \"Flow1 has no upload date that allows us to compare age of children.\"\n                    upload_date_current_flow = dateutil.parser.parse(flow1.upload_date)\n                    upload_date_parent_flow = dateutil.parser.parse(\n                        ignore_parameter_values_on_older_children,\n                    )\n                    if upload_date_current_flow &lt; upload_date_parent_flow:\n                        continue\n\n                if ignore_parameter_values:\n                    # Continue needs to be done here as the first if\n                    # statement triggers in both special cases\n                    continue\n            elif (\n                key == \"custom_name\"\n                and ignore_custom_name_if_none\n                and (attr1 is None or attr2 is None)\n            ):\n                # If specified, we allow `custom_name` inequality if one flow's name is None.\n                # Helps with backwards compatibility as `custom_name` is now auto-generated, but\n                # before it used to be `None`.\n                continue\n            elif key == \"parameters_meta_info\":\n                # this value is a dictionary where each key is a parameter name, containing another\n                # dictionary with keys specifying the parameter's 'description' and 'data_type'\n                # checking parameter descriptions can be ignored since that might change\n                # data type check can also be ignored if one of them is not defined, i.e., None\n                params1 = set(flow1.parameters_meta_info)\n                params2 = set(flow2.parameters_meta_info)\n                if params1 != params2:\n                    raise ValueError(\n                        \"Parameter list in meta info for parameters differ \" \"in the two flows.\",\n                    )\n                # iterating over the parameter's meta info list\n                for param in params1:\n                    if (\n                        isinstance(flow1.parameters_meta_info[param], Dict)\n                        and isinstance(flow2.parameters_meta_info[param], Dict)\n                        and \"data_type\" in flow1.parameters_meta_info[param]\n                        and \"data_type\" in flow2.parameters_meta_info[param]\n                    ):\n                        value1 = flow1.parameters_meta_info[param][\"data_type\"]\n                        value2 = flow2.parameters_meta_info[param][\"data_type\"]\n                    else:\n                        value1 = flow1.parameters_meta_info[param]\n                        value2 = flow2.parameters_meta_info[param]\n                    if value1 is None or value2 is None:\n                        continue\n\n                    if value1 != value2:\n                        raise ValueError(\n                            f\"Flow {flow1.name}: data type for parameter {param} in {key} differ \"\n                            f\"as {value1}\\nvs\\n{value2}\",\n                        )\n                # the continue is to avoid the 'attr != attr2' check at end of function\n                continue\n\n            if attr1 != attr2:\n                raise ValueError(\n                    f\"Flow {flow1.name!s}: values for attribute '{key!s}' differ: \"\n                    f\"'{attr1!s}'\\nvs\\n'{attr2!s}'.\",\n                )\n</code></pre>"},{"location":"reference/flows/#openml.flows.delete_flow","title":"<code>delete_flow(flow_id)</code>","text":"<p>Delete flow with id <code>flow_id</code> from the OpenML server.</p> <p>You can only delete flows which you uploaded and which which are not linked to runs.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>OpenML id of the flow</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def delete_flow(flow_id: int) -&gt; bool:\n    \"\"\"Delete flow with id `flow_id` from the OpenML server.\n\n    You can only delete flows which you uploaded and which\n    which are not linked to runs.\n\n    Parameters\n    ----------\n    flow_id : int\n        OpenML id of the flow\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"flow\", flow_id)\n</code></pre>"},{"location":"reference/flows/#openml.flows.flow_exists","title":"<code>flow_exists(name, external_version)</code>","text":"<p>Retrieves the flow id.</p> <p>A flow is uniquely identified by name + external_version.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>string</code> <p>Name of the flow</p> required <code>external_version</code> <code>string</code> <p>Version information associated with flow.</p> required <p>Returns:</p> Name Type Description <code>flow_exist</code> <code>int or bool</code> <p>flow id iff exists, False otherwise</p> Notes <p>see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def flow_exists(name: str, external_version: str) -&gt; int | bool:\n    \"\"\"Retrieves the flow id.\n\n    A flow is uniquely identified by name + external_version.\n\n    Parameters\n    ----------\n    name : string\n        Name of the flow\n    external_version : string\n        Version information associated with flow.\n\n    Returns\n    -------\n    flow_exist : int or bool\n        flow id iff exists, False otherwise\n\n    Notes\n    -----\n    see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version\n    \"\"\"\n    if not (isinstance(name, str) and len(name) &gt; 0):\n        raise ValueError(\"Argument 'name' should be a non-empty string\")\n    if not (isinstance(name, str) and len(external_version) &gt; 0):\n        raise ValueError(\"Argument 'version' should be a non-empty string\")\n\n    xml_response = openml._api_calls._perform_api_call(\n        \"flow/exists\",\n        \"post\",\n        data={\"name\": name, \"external_version\": external_version},\n    )\n\n    result_dict = xmltodict.parse(xml_response)\n    flow_id = int(result_dict[\"oml:flow_exists\"][\"oml:id\"])\n    return flow_id if flow_id &gt; 0 else False\n</code></pre>"},{"location":"reference/flows/#openml.flows.get_flow","title":"<code>get_flow(flow_id, reinstantiate=False, strict_version=True)</code>","text":"<p>Download the OpenML flow for a given flow ID.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>The OpenML flow id.</p> required <code>reinstantiate</code> <code>bool</code> <p>Whether to reinstantiate the flow to a model instance.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>flow</code> <code>OpenMLFlow</code> <p>the flow</p> Source code in <code>openml/flows/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_flow(flow_id: int, reinstantiate: bool = False, strict_version: bool = True) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Download the OpenML flow for a given flow ID.\n\n    Parameters\n    ----------\n    flow_id : int\n        The OpenML flow id.\n\n    reinstantiate: bool\n        Whether to reinstantiate the flow to a model instance.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    flow : OpenMLFlow\n        the flow\n    \"\"\"\n    flow_id = int(flow_id)\n    flow = _get_flow_description(flow_id)\n\n    if reinstantiate:\n        flow.model = flow.extension.flow_to_model(flow, strict_version=strict_version)\n        if not strict_version:\n            # check if we need to return a new flow b/c of version mismatch\n            new_flow = flow.extension.model_to_flow(flow.model)\n            if new_flow.dependencies != flow.dependencies:\n                return new_flow\n    return flow\n</code></pre>"},{"location":"reference/flows/#openml.flows.get_flow_id","title":"<code>get_flow_id(model=None, name=None, exact_version=True)</code>","text":"<p>Retrieves the flow id for a model or a flow name.</p> <p>Provide either a model or a name to this function. Depending on the input, it does</p> <ul> <li><code>model</code> and <code>exact_version == True</code>: This helper function first queries for the necessary   extension. Second, it uses that extension to convert the model into a flow. Third, it   executes <code>flow_exists</code> to potentially obtain the flow id the flow is published to the   server.</li> <li><code>model</code> and <code>exact_version == False</code>: This helper function first queries for the   necessary extension. Second, it uses that extension to convert the model into a flow. Third   it calls <code>list_flows</code> and filters the returned values based on the flow name.</li> <li><code>name</code>: Ignores <code>exact_version</code> and calls <code>list_flows</code>, then filters the returned   values based on the flow name.</li> </ul> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>object</code> <p>Any model. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>name</code> <code>str</code> <p>Name of the flow. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>exact_version</code> <code>bool</code> <p>Whether to return the flow id of the exact version or all flow ids where the name of the flow matches. This is only taken into account for a model where a version number is available (requires <code>model</code> to be set).</p> <code>True</code> <p>Returns:</p> Type Description <code>(int or bool, List)</code> <p>flow id iff exists, <code>False</code> otherwise, List if <code>exact_version is False</code></p> Source code in <code>openml/flows/functions.py</code> <pre><code>def get_flow_id(\n    model: Any | None = None,\n    name: str | None = None,\n    exact_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; int | bool | list[int]:\n    \"\"\"Retrieves the flow id for a model or a flow name.\n\n    Provide either a model or a name to this function. Depending on the input, it does\n\n    * ``model`` and ``exact_version == True``: This helper function first queries for the necessary\n      extension. Second, it uses that extension to convert the model into a flow. Third, it\n      executes ``flow_exists`` to potentially obtain the flow id the flow is published to the\n      server.\n    * ``model`` and ``exact_version == False``: This helper function first queries for the\n      necessary extension. Second, it uses that extension to convert the model into a flow. Third\n      it calls ``list_flows`` and filters the returned values based on the flow name.\n    * ``name``: Ignores ``exact_version`` and calls ``list_flows``, then filters the returned\n      values based on the flow name.\n\n    Parameters\n    ----------\n    model : object\n        Any model. Must provide either ``model`` or ``name``.\n    name : str\n        Name of the flow. Must provide either ``model`` or ``name``.\n    exact_version : bool\n        Whether to return the flow id of the exact version or all flow ids where the name\n        of the flow matches. This is only taken into account for a model where a version number\n        is available (requires ``model`` to be set).\n\n    Returns\n    -------\n    int or bool, List\n        flow id iff exists, ``False`` otherwise, List if ``exact_version is False``\n    \"\"\"\n    if model is not None and name is not None:\n        raise ValueError(\"Must provide either argument `model` or argument `name`, but not both.\")\n\n    if model is not None:\n        extension = openml.extensions.get_extension_by_model(model, raise_if_no_extension=True)\n        if extension is None:\n            # This should never happen and is only here to please mypy will be gone soon once the\n            # whole function is removed\n            raise TypeError(extension)\n        flow = extension.model_to_flow(model)\n        flow_name = flow.name\n        external_version = flow.external_version\n    elif name is not None:\n        flow_name = name\n        exact_version = False\n        external_version = None\n    else:\n        raise ValueError(\n            \"Need to provide either argument `model` or argument `name`, but both are `None`.\"\n        )\n\n    if exact_version:\n        if external_version is None:\n            raise ValueError(\"exact_version should be False if model is None!\")\n        return flow_exists(name=flow_name, external_version=external_version)\n\n    flows = list_flows(output_format=\"dataframe\")\n    assert isinstance(flows, pd.DataFrame)  # Make mypy happy\n    flows = flows.query(f'name == \"{flow_name}\"')\n    return flows[\"id\"].to_list()  # type: ignore[no-any-return]\n</code></pre>"},{"location":"reference/flows/#openml.flows.list_flows","title":"<code>list_flows(offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; dict\n</code></pre><pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_flows(offset: int | None, size: int | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all flows which are on OpenML. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of flows to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of flows to return</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: uploader.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>flows</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from flow_id to a dict giving a brief overview of the     respective flow.     Every flow is represented by a dictionary containing     the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> </ul> Source code in <code>openml/flows/functions.py</code> <pre><code>def list_flows(\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all flows which are on OpenML.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of flows to skip, starting from the first\n    size : int, optional\n        the maximum number of flows to return\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: uploader.\n\n    Returns\n    -------\n    flows : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from flow_id to a dict giving a brief overview of the\n            respective flow.\n            Every flow is represented by a dictionary containing\n            the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(\n        list_output_format=output_format,\n        listing_call=_list_flows,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/flows/flow/","title":"flow","text":""},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow","title":"<code>OpenMLFlow</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Flow. Stores machine learning models.</p> <p>Flows should not be generated manually, but by the function :meth:<code>openml.flows.create_flow_from_model</code>. Using this helper function ensures that all relevant fields are filled in.</p> <p>Implements <code>openml.implementation.upload.xsd &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/ openml.implementation.upload.xsd&gt;</code>_.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the flow. Is used together with the attribute <code>external_version</code> as a unique identifier of the flow.</p> required <code>description</code> <code>str</code> <p>Human-readable description of the flow (free text).</p> required <code>model</code> <code>object</code> <p>ML model which is described by this flow.</p> required <code>components</code> <code>OrderedDict</code> <p>Mapping from component identifier to an OpenMLFlow object. Components are usually subfunctions of an algorithm (e.g. kernels), base learners in ensemble algorithms (decision tree in adaboost) or building blocks of a machine learning pipeline. Components are modeled as independent flows and can be shared between flows (different pipelines can use the same components).</p> required <code>parameters</code> <code>OrderedDict</code> <p>Mapping from parameter name to the parameter default value. The parameter default value must be of type <code>str</code>, so that the respective toolbox plugin can take care of casting the parameter default value to the correct type.</p> required <code>parameters_meta_info</code> <code>OrderedDict</code> <p>Mapping from parameter name to <code>dict</code>. Stores additional information for each parameter. Required keys are <code>data_type</code> and <code>description</code>.</p> required <code>external_version</code> <code>str</code> <p>Version number of the software the flow is implemented in. Is used together with the attribute <code>name</code> as a uniquer identifier of the flow.</p> required <code>tags</code> <code>list</code> <p>List of tags. Created on the server by other API calls.</p> required <code>language</code> <code>str</code> <p>Natural language the flow is described in (not the programming language).</p> required <code>dependencies</code> <code>str</code> <p>A list of dependencies necessary to run the flow. This field should contain all libraries the flow depends on. To allow reproducibility it should also specify the exact version numbers.</p> required <code>class_name</code> <code>str</code> <p>The development language name of the class which is described by this flow.</p> <code>None</code> <code>custom_name</code> <code>str</code> <p>Custom name of the flow given by the owner.</p> <code>None</code> <code>binary_url</code> <code>str</code> <p>Url from which the binary can be downloaded. Added by the server. Ignored when uploaded manually. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_format</code> <code>str</code> <p>Format in which the binary code was uploaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_md5</code> <code>str</code> <p>MD5 checksum to check if the binary code was correctly downloaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>uploader</code> <code>str</code> <p>OpenML user ID of the uploader. Filled in by the server.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>Date the flow was uploaded. Filled in by the server.</p> <code>None</code> <code>flow_id</code> <code>int</code> <p>Flow ID. Assigned by the server.</p> <code>None</code> <code>extension</code> <code>Extension</code> <p>The extension for a flow (e.g., sklearn).</p> <code>None</code> <code>version</code> <code>str</code> <p>OpenML version of the flow. Assigned by the server.</p> <code>None</code> Source code in <code>openml/flows/flow.py</code> <pre><code>class OpenMLFlow(OpenMLBase):\n    \"\"\"OpenML Flow. Stores machine learning models.\n\n    Flows should not be generated manually, but by the function\n    :meth:`openml.flows.create_flow_from_model`. Using this helper function\n    ensures that all relevant fields are filled in.\n\n    Implements `openml.implementation.upload.xsd\n    &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/\n    openml.implementation.upload.xsd&gt;`_.\n\n    Parameters\n    ----------\n    name : str\n        Name of the flow. Is used together with the attribute\n        `external_version` as a unique identifier of the flow.\n    description : str\n        Human-readable description of the flow (free text).\n    model : object\n        ML model which is described by this flow.\n    components : OrderedDict\n        Mapping from component identifier to an OpenMLFlow object. Components\n        are usually subfunctions of an algorithm (e.g. kernels), base learners\n        in ensemble algorithms (decision tree in adaboost) or building blocks\n        of a machine learning pipeline. Components are modeled as independent\n        flows and can be shared between flows (different pipelines can use\n        the same components).\n    parameters : OrderedDict\n        Mapping from parameter name to the parameter default value. The\n        parameter default value must be of type `str`, so that the respective\n        toolbox plugin can take care of casting the parameter default value to\n        the correct type.\n    parameters_meta_info : OrderedDict\n        Mapping from parameter name to `dict`. Stores additional information\n        for each parameter. Required keys are `data_type` and `description`.\n    external_version : str\n        Version number of the software the flow is implemented in. Is used\n        together with the attribute `name` as a uniquer identifier of the flow.\n    tags : list\n        List of tags. Created on the server by other API calls.\n    language : str\n        Natural language the flow is described in (not the programming\n        language).\n    dependencies : str\n        A list of dependencies necessary to run the flow. This field should\n        contain all libraries the flow depends on. To allow reproducibility\n        it should also specify the exact version numbers.\n    class_name : str, optional\n        The development language name of the class which is described by this\n        flow.\n    custom_name : str, optional\n        Custom name of the flow given by the owner.\n    binary_url : str, optional\n        Url from which the binary can be downloaded. Added by the server.\n        Ignored when uploaded manually. Will not be used by the python API\n        because binaries aren't compatible across machines.\n    binary_format : str, optional\n        Format in which the binary code was uploaded. Will not be used by the\n        python API because binaries aren't compatible across machines.\n    binary_md5 : str, optional\n        MD5 checksum to check if the binary code was correctly downloaded. Will\n        not be used by the python API because binaries aren't compatible across\n        machines.\n    uploader : str, optional\n        OpenML user ID of the uploader. Filled in by the server.\n    upload_date : str, optional\n        Date the flow was uploaded. Filled in by the server.\n    flow_id : int, optional\n        Flow ID. Assigned by the server.\n    extension : Extension, optional\n        The extension for a flow (e.g., sklearn).\n    version : str, optional\n        OpenML version of the flow. Assigned by the server.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        name: str,\n        description: str,\n        model: object,\n        components: dict,\n        parameters: dict,\n        parameters_meta_info: dict,\n        external_version: str,\n        tags: list,\n        language: str,\n        dependencies: str,\n        class_name: str | None = None,\n        custom_name: str | None = None,\n        binary_url: str | None = None,\n        binary_format: str | None = None,\n        binary_md5: str | None = None,\n        uploader: str | None = None,\n        upload_date: str | None = None,\n        flow_id: int | None = None,\n        extension: Extension | None = None,\n        version: str | None = None,\n    ):\n        self.name = name\n        self.description = description\n        self.model = model\n\n        for variable, variable_name in [\n            [components, \"components\"],\n            [parameters, \"parameters\"],\n            [parameters_meta_info, \"parameters_meta_info\"],\n        ]:\n            if not isinstance(variable, (OrderedDict, dict)):\n                raise TypeError(\n                    f\"{variable_name} must be of type OrderedDict or dict, \"\n                    f\"but is {type(variable)}.\",\n                )\n\n        self.components = components\n        self.parameters = parameters\n        self.parameters_meta_info = parameters_meta_info\n        self.class_name = class_name\n\n        keys_parameters = set(parameters.keys())\n        keys_parameters_meta_info = set(parameters_meta_info.keys())\n        if len(keys_parameters.difference(keys_parameters_meta_info)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters, but not in \"\n                \"parameters_meta_info.\"\n                % str(keys_parameters.difference(keys_parameters_meta_info)),\n            )\n        if len(keys_parameters_meta_info.difference(keys_parameters)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters_meta_info, \"\n                \"but not in parameters.\"\n                % str(keys_parameters_meta_info.difference(keys_parameters)),\n            )\n\n        self.external_version = external_version\n        self.uploader = uploader\n\n        self.custom_name = custom_name\n        self.tags = tags if tags is not None else []\n        self.binary_url = binary_url\n        self.binary_format = binary_format\n        self.binary_md5 = binary_md5\n        self.version = version\n        self.upload_date = upload_date\n        self.language = language\n        self.dependencies = dependencies\n        self.flow_id = flow_id\n        if extension is None:\n            self._extension = get_extension_by_flow(self)\n        else:\n            self._extension = extension\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the flow.\"\"\"\n        return self.flow_id\n\n    @property\n    def extension(self) -&gt; Extension:\n        \"\"\"The extension of the flow (e.g., sklearn).\"\"\"\n        if self._extension is not None:\n            return self._extension\n\n        raise RuntimeError(\n            f\"No extension could be found for flow {self.flow_id}: {self.name}\",\n        )\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields = {\n            \"Flow Name\": self.name,\n            \"Flow Description\": self.description,\n            \"Dependencies\": self.dependencies,\n        }\n        if self.flow_id is not None:\n            fields[\"Flow URL\"] = self.openml_url if self.openml_url is not None else \"None\"\n            fields[\"Flow ID\"] = str(self.flow_id)\n            if self.version is not None:\n                fields[\"Flow ID\"] += f\" (version {self.version})\"\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.binary_url is not None:\n            fields[\"Binary URL\"] = self.binary_url\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Flow ID\",\n            \"Flow URL\",\n            \"Flow Name\",\n            \"Flow Description\",\n            \"Binary URL\",\n            \"Upload Date\",\n            \"Dependencies\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: C901, PLR0912\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        flow_container = OrderedDict()  # type: 'dict[str, dict]'\n        flow_dict = OrderedDict(\n            [(\"@xmlns:oml\", \"http://openml.org/openml\")],\n        )  # type: 'dict[str, list | str]'  # E501\n        flow_container[\"oml:flow\"] = flow_dict\n        _add_if_nonempty(flow_dict, \"oml:id\", self.flow_id)\n\n        for required in [\"name\", \"external_version\"]:\n            if getattr(self, required) is None:\n                raise ValueError(f\"self.{required} is required but None\")\n        for attribute in [\n            \"uploader\",\n            \"name\",\n            \"custom_name\",\n            \"class_name\",\n            \"version\",\n            \"external_version\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n        ]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        if not self.description:\n            logger = logging.getLogger(__name__)\n            logger.warning(\"Flow % has empty description\", self.name)\n\n        flow_parameters = []\n        for key in self.parameters:\n            param_dict = OrderedDict()  # type: 'OrderedDict[str, str]'\n            param_dict[\"oml:name\"] = key\n            meta_info = self.parameters_meta_info[key]\n\n            _add_if_nonempty(param_dict, \"oml:data_type\", meta_info[\"data_type\"])\n            param_dict[\"oml:default_value\"] = self.parameters[key]\n            _add_if_nonempty(param_dict, \"oml:description\", meta_info[\"description\"])\n\n            for key_, value in param_dict.items():\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n                if value is not None and not isinstance(value, str):\n                    raise ValueError(\n                        f\"Parameter value {value} cannot be serialized \"\n                        f\"because it is of type {type(value)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            flow_parameters.append(param_dict)\n\n        flow_dict[\"oml:parameter\"] = flow_parameters\n\n        components = []\n        for key in self.components:\n            component_dict = OrderedDict()  # type: 'OrderedDict[str, dict]'\n            component_dict[\"oml:identifier\"] = key\n            if self.components[key] in [\"passthrough\", \"drop\"]:\n                component_dict[\"oml:flow\"] = {\n                    \"oml-python:serialized_object\": \"component_reference\",\n                    \"value\": {\"key\": self.components[key], \"step_name\": self.components[key]},\n                }\n            else:\n                component_dict[\"oml:flow\"] = self.components[key]._to_dict()[\"oml:flow\"]\n\n            for key_ in component_dict:\n                # We only need to check if the key is a string, because the\n                # value is a flow. The flow itself is valid by recursion\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            components.append(component_dict)\n\n        flow_dict[\"oml:component\"] = components\n        flow_dict[\"oml:tag\"] = self.tags\n        for attribute in [\"binary_url\", \"binary_format\", \"binary_md5\"]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        return flow_container\n\n    @classmethod\n    def _from_dict(cls, xml_dict: dict) -&gt; OpenMLFlow:\n        \"\"\"Create a flow from an xml description.\n\n        Calls itself recursively to create :class:`OpenMLFlow` objects of\n        subflows (components).\n\n        XML definition of a flow is available at\n        https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd\n\n        Parameters\n        ----------\n        xml_dict : dict\n            Dictionary representation of the flow as created by _to_dict()\n\n        Returns\n        -------\n            OpenMLFlow\n\n        \"\"\"  # E501\n        arguments = OrderedDict()\n        dic = xml_dict[\"oml:flow\"]\n\n        # Mandatory parts in the xml file\n        for key in [\"name\"]:\n            arguments[key] = dic[\"oml:\" + key]\n\n        # non-mandatory parts in the xml file\n        for key in [\n            \"external_version\",\n            \"uploader\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n            \"version\",\n            \"binary_url\",\n            \"binary_format\",\n            \"binary_md5\",\n            \"class_name\",\n            \"custom_name\",\n        ]:\n            arguments[key] = dic.get(\"oml:\" + key)\n\n        # has to be converted to an int if present and cannot parsed in the\n        # two loops above\n        arguments[\"flow_id\"] = int(dic[\"oml:id\"]) if dic.get(\"oml:id\") is not None else None\n\n        # Now parse parts of a flow which can occur multiple times like\n        # parameters, components (subflows) and tags. These can't be tackled\n        # in the loops above because xmltodict returns a dict if such an\n        # entity occurs once, and a list if it occurs multiple times.\n        # Furthermore, they must be treated differently, for example\n        # for components this method is called recursively and\n        # for parameters the actual information is split into two dictionaries\n        # for easier access in python.\n\n        parameters = OrderedDict()\n        parameters_meta_info = OrderedDict()\n        if \"oml:parameter\" in dic:\n            # In case of a single parameter, xmltodict returns a dictionary,\n            # otherwise a list.\n            oml_parameters = extract_xml_tags(\"oml:parameter\", dic, allow_none=False)\n\n            for oml_parameter in oml_parameters:\n                parameter_name = oml_parameter[\"oml:name\"]\n                default_value = oml_parameter[\"oml:default_value\"]\n                parameters[parameter_name] = default_value\n\n                meta_info = OrderedDict()\n                meta_info[\"description\"] = oml_parameter.get(\"oml:description\")\n                meta_info[\"data_type\"] = oml_parameter.get(\"oml:data_type\")\n                parameters_meta_info[parameter_name] = meta_info\n        arguments[\"parameters\"] = parameters\n        arguments[\"parameters_meta_info\"] = parameters_meta_info\n\n        components = OrderedDict()\n        if \"oml:component\" in dic:\n            # In case of a single component xmltodict returns a dict,\n            # otherwise a list.\n            oml_components = extract_xml_tags(\"oml:component\", dic, allow_none=False)\n\n            for component in oml_components:\n                flow = OpenMLFlow._from_dict(component)\n                components[component[\"oml:identifier\"]] = flow\n        arguments[\"components\"] = components\n        arguments[\"tags\"] = extract_xml_tags(\"oml:tag\", dic)\n\n        arguments[\"model\"] = None\n        return cls(**arguments)\n\n    def to_filesystem(self, output_directory: str | Path) -&gt; None:\n        \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n        output_directory = Path(output_directory)\n        output_directory.mkdir(parents=True, exist_ok=True)\n\n        output_path = output_directory / \"flow.xml\"\n        if output_path.exists():\n            raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n        run_xml = self._to_xml()\n        with output_path.open(\"w\") as f:\n            f.write(run_xml)\n\n    @classmethod\n    def from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n        \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n        input_directory = Path(input_directory) / \"flow.xml\"\n        with input_directory.open() as f:\n            xml_string = f.read()\n        return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.flow_id = int(xml_response[\"oml:upload_flow\"][\"oml:id\"])\n\n    def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n        \"\"\"Publish this flow to OpenML server.\n\n        Raises a PyOpenMLError if the flow exists on the server, but\n        `self.flow_id` does not match the server known flow id.\n\n        Parameters\n        ----------\n        raise_error_if_exists : bool, optional (default=False)\n            If True, raise PyOpenMLError if the flow exists on the server.\n            If False, update the local flow to match the server flow.\n\n        Returns\n        -------\n        self : OpenMLFlow\n\n        \"\"\"\n        # Import at top not possible because of cyclic dependencies. In\n        # particular, flow.py tries to import functions.py in order to call\n        # get_flow(), while functions.py tries to import flow.py in order to\n        # instantiate an OpenMLFlow.\n        import openml.flows.functions\n\n        flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n        if not flow_id:\n            if self.flow_id:\n                raise openml.exceptions.PyOpenMLError(\n                    \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n                )\n            super().publish()\n            assert self.flow_id is not None  # for mypy\n            flow_id = self.flow_id\n        elif raise_error_if_exists:\n            error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n            raise openml.exceptions.PyOpenMLError(error_message)\n        elif self.flow_id is not None and self.flow_id != flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n            )\n\n        flow = openml.flows.functions.get_flow(flow_id)\n        _copy_server_fields(flow, self)\n        try:\n            openml.flows.functions.assert_flows_equal(\n                self,\n                flow,\n                flow.upload_date,\n                ignore_parameter_values=True,\n                ignore_custom_name_if_none=True,\n            )\n        except ValueError as e:\n            message = e.args[0]\n            raise ValueError(\n                \"The flow on the server is inconsistent with the local flow. \"\n                f\"The server flow ID is {flow_id}. Please check manually and remove \"\n                f\"the flow if necessary! Error is:\\n'{message}'\",\n            ) from e\n        return self\n\n    def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n        \"\"\"\n        Returns for each sub-component of the flow the path of identifiers\n        that should be traversed to reach this component. The resulting dict\n        maps a key (identifying a flow by either its id, name or fullname) to\n        the parameter prefix.\n\n        Parameters\n        ----------\n        key_item: str\n            The flow attribute that will be used to identify flows in the\n            structure. Allowed values {flow_id, name}\n\n        Returns\n        -------\n        dict[str, List[str]]\n            The flow structure\n        \"\"\"\n        if key_item not in [\"flow_id\", \"name\"]:\n            raise ValueError(\"key_item should be in {flow_id, name}\")\n        structure = {}\n        for key, sub_flow in self.components.items():\n            sub_structure = sub_flow.get_structure(key_item)\n            for flow_name, flow_sub_structure in sub_structure.items():\n                structure[flow_name] = [key, *flow_sub_structure]\n        structure[getattr(self, key_item)] = []\n        return structure\n\n    def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n        \"\"\"\n        Returns a subflow from the tree of dependencies.\n\n        Parameters\n        ----------\n        structure: list[str]\n            A list of strings, indicating the location of the subflow\n\n        Returns\n        -------\n        OpenMLFlow\n            The OpenMLFlow that corresponds to the structure\n        \"\"\"\n        # make a copy of structure, as we don't want to change it in the\n        # outer scope\n        structure = list(structure)\n        if len(structure) &lt; 1:\n            raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n        sub_identifier = structure[0]\n        if sub_identifier not in self.components:\n            raise ValueError(\n                f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n            )\n        if len(structure) == 1:\n            return self.components[sub_identifier]  # type: ignore\n\n        structure.pop(0)\n        return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.extension","title":"<code>extension: Extension</code>  <code>property</code>","text":"<p>The extension of the flow (e.g., sklearn).</p>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the flow.</p>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.from_filesystem","title":"<code>from_filesystem(input_directory)</code>  <code>classmethod</code>","text":"<p>Read a flow from an XML in input_directory on the filesystem.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n    \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n    input_directory = Path(input_directory) / \"flow.xml\"\n    with input_directory.open() as f:\n        xml_string = f.read()\n    return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.get_structure","title":"<code>get_structure(key_item)</code>","text":"<p>Returns for each sub-component of the flow the path of identifiers that should be traversed to reach this component. The resulting dict maps a key (identifying a flow by either its id, name or fullname) to the parameter prefix.</p> <p>Parameters:</p> Name Type Description Default <code>key_item</code> <code>str</code> <p>The flow attribute that will be used to identify flows in the structure. Allowed values {flow_id, name}</p> required <p>Returns:</p> Type Description <code>dict[str, List[str]]</code> <p>The flow structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n    \"\"\"\n    Returns for each sub-component of the flow the path of identifiers\n    that should be traversed to reach this component. The resulting dict\n    maps a key (identifying a flow by either its id, name or fullname) to\n    the parameter prefix.\n\n    Parameters\n    ----------\n    key_item: str\n        The flow attribute that will be used to identify flows in the\n        structure. Allowed values {flow_id, name}\n\n    Returns\n    -------\n    dict[str, List[str]]\n        The flow structure\n    \"\"\"\n    if key_item not in [\"flow_id\", \"name\"]:\n        raise ValueError(\"key_item should be in {flow_id, name}\")\n    structure = {}\n    for key, sub_flow in self.components.items():\n        sub_structure = sub_flow.get_structure(key_item)\n        for flow_name, flow_sub_structure in sub_structure.items():\n            structure[flow_name] = [key, *flow_sub_structure]\n    structure[getattr(self, key_item)] = []\n    return structure\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.get_subflow","title":"<code>get_subflow(structure)</code>","text":"<p>Returns a subflow from the tree of dependencies.</p> <p>Parameters:</p> Name Type Description Default <code>structure</code> <code>list[str]</code> <p>A list of strings, indicating the location of the subflow</p> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> <p>The OpenMLFlow that corresponds to the structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n    \"\"\"\n    Returns a subflow from the tree of dependencies.\n\n    Parameters\n    ----------\n    structure: list[str]\n        A list of strings, indicating the location of the subflow\n\n    Returns\n    -------\n    OpenMLFlow\n        The OpenMLFlow that corresponds to the structure\n    \"\"\"\n    # make a copy of structure, as we don't want to change it in the\n    # outer scope\n    structure = list(structure)\n    if len(structure) &lt; 1:\n        raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n    sub_identifier = structure[0]\n    if sub_identifier not in self.components:\n        raise ValueError(\n            f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n        )\n    if len(structure) == 1:\n        return self.components[sub_identifier]  # type: ignore\n\n    structure.pop(0)\n    return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.publish","title":"<code>publish(raise_error_if_exists=False)</code>","text":"<p>Publish this flow to OpenML server.</p> <p>Raises a PyOpenMLError if the flow exists on the server, but <code>self.flow_id</code> does not match the server known flow id.</p> <p>Parameters:</p> Name Type Description Default <code>raise_error_if_exists</code> <code>(bool, optional(default=False))</code> <p>If True, raise PyOpenMLError if the flow exists on the server. If False, update the local flow to match the server flow.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>self</code> <code>OpenMLFlow</code> Source code in <code>openml/flows/flow.py</code> <pre><code>def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Publish this flow to OpenML server.\n\n    Raises a PyOpenMLError if the flow exists on the server, but\n    `self.flow_id` does not match the server known flow id.\n\n    Parameters\n    ----------\n    raise_error_if_exists : bool, optional (default=False)\n        If True, raise PyOpenMLError if the flow exists on the server.\n        If False, update the local flow to match the server flow.\n\n    Returns\n    -------\n    self : OpenMLFlow\n\n    \"\"\"\n    # Import at top not possible because of cyclic dependencies. In\n    # particular, flow.py tries to import functions.py in order to call\n    # get_flow(), while functions.py tries to import flow.py in order to\n    # instantiate an OpenMLFlow.\n    import openml.flows.functions\n\n    flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n    if not flow_id:\n        if self.flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n            )\n        super().publish()\n        assert self.flow_id is not None  # for mypy\n        flow_id = self.flow_id\n    elif raise_error_if_exists:\n        error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n        raise openml.exceptions.PyOpenMLError(error_message)\n    elif self.flow_id is not None and self.flow_id != flow_id:\n        raise openml.exceptions.PyOpenMLError(\n            \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n        )\n\n    flow = openml.flows.functions.get_flow(flow_id)\n    _copy_server_fields(flow, self)\n    try:\n        openml.flows.functions.assert_flows_equal(\n            self,\n            flow,\n            flow.upload_date,\n            ignore_parameter_values=True,\n            ignore_custom_name_if_none=True,\n        )\n    except ValueError as e:\n        message = e.args[0]\n        raise ValueError(\n            \"The flow on the server is inconsistent with the local flow. \"\n            f\"The server flow ID is {flow_id}. Please check manually and remove \"\n            f\"the flow if necessary! Error is:\\n'{message}'\",\n        ) from e\n    return self\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.to_filesystem","title":"<code>to_filesystem(output_directory)</code>","text":"<p>Write a flow to the filesystem as XML to output_directory.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def to_filesystem(self, output_directory: str | Path) -&gt; None:\n    \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n    output_directory = Path(output_directory)\n    output_directory.mkdir(parents=True, exist_ok=True)\n\n    output_path = output_directory / \"flow.xml\"\n    if output_path.exists():\n        raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n    run_xml = self._to_xml()\n    with output_path.open(\"w\") as f:\n        f.write(run_xml)\n</code></pre>"},{"location":"reference/flows/functions/","title":"functions","text":""},{"location":"reference/flows/functions/#openml.flows.functions.__list_flows","title":"<code>__list_flows(api_call, output_format='dict')</code>","text":"<pre><code>__list_flows(api_call: str, output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>__list_flows(api_call: str, output_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Retrieve information about flows from OpenML API and parse it to a dictionary or a Pandas DataFrame.</p> <p>Parameters:</p> Name Type Description Default <code>api_call</code> <code>str</code> <p>Retrieves the information about flows.</p> required <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The output format.</p> <code>'dict'</code> <p>Returns:</p> Type Description <code>    The flows information in the specified output format.</code> Source code in <code>openml/flows/functions.py</code> <pre><code>def __list_flows(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\"] = \"dict\"\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Retrieve information about flows from OpenML API\n    and parse it to a dictionary or a Pandas DataFrame.\n\n    Parameters\n    ----------\n    api_call: str\n        Retrieves the information about flows.\n    output_format: str in {\"dict\", \"dataframe\"}\n        The output format.\n\n    Returns\n    -------\n        The flows information in the specified output format.\n    \"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    flows_dict = xmltodict.parse(xml_string, force_list=(\"oml:flow\",))\n\n    # Minimalistic check if the XML is useful\n    assert isinstance(flows_dict[\"oml:flows\"][\"oml:flow\"], list), type(flows_dict[\"oml:flows\"])\n    assert flows_dict[\"oml:flows\"][\"@xmlns:oml\"] == \"http://openml.org/openml\", flows_dict[\n        \"oml:flows\"\n    ][\"@xmlns:oml\"]\n\n    flows = {}\n    for flow_ in flows_dict[\"oml:flows\"][\"oml:flow\"]:\n        fid = int(flow_[\"oml:id\"])\n        flow = {\n            \"id\": fid,\n            \"full_name\": flow_[\"oml:full_name\"],\n            \"name\": flow_[\"oml:name\"],\n            \"version\": flow_[\"oml:version\"],\n            \"external_version\": flow_[\"oml:external_version\"],\n            \"uploader\": flow_[\"oml:uploader\"],\n        }\n        flows[fid] = flow\n\n    if output_format == \"dataframe\":\n        flows = pd.DataFrame.from_dict(flows, orient=\"index\")\n\n    return flows\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.assert_flows_equal","title":"<code>assert_flows_equal(flow1, flow2, ignore_parameter_values_on_older_children=None, ignore_parameter_values=False, ignore_custom_name_if_none=False, check_description=True)</code>","text":"<p>Check equality of two flows.</p> <p>Two flows are equal if their all keys which are not set by the server are equal, as well as all their parameters and components.</p> <p>Parameters:</p> Name Type Description Default <code>flow1</code> <code>OpenMLFlow</code> required <code>flow2</code> <code>OpenMLFlow</code> required <code>ignore_parameter_values_on_older_children</code> <code>str(optional)</code> <p>If set to <code>OpenMLFlow.upload_date</code>, ignores parameters in a child flow if it's upload date predates the upload date of the parent flow.</p> <code>None</code> <code>ignore_parameter_values</code> <code>bool</code> <p>Whether to ignore parameter values when comparing flows.</p> <code>False</code> <code>ignore_custom_name_if_none</code> <code>bool</code> <p>Whether to ignore the custom name field if either flow has <code>custom_name</code> equal to <code>None</code>.</p> <code>False</code> <code>check_description</code> <code>bool</code> <p>Whether to ignore matching of flow descriptions.</p> <code>True</code> Source code in <code>openml/flows/functions.py</code> <pre><code>def assert_flows_equal(  # noqa: C901, PLR0912, PLR0913, PLR0915\n    flow1: OpenMLFlow,\n    flow2: OpenMLFlow,\n    ignore_parameter_values_on_older_children: str | None = None,\n    ignore_parameter_values: bool = False,  # noqa: FBT001, FBT002\n    ignore_custom_name_if_none: bool = False,  # noqa:  FBT001, FBT002\n    check_description: bool = True,  # noqa:  FBT001, FBT002\n) -&gt; None:\n    \"\"\"Check equality of two flows.\n\n    Two flows are equal if their all keys which are not set by the server\n    are equal, as well as all their parameters and components.\n\n    Parameters\n    ----------\n    flow1 : OpenMLFlow\n\n    flow2 : OpenMLFlow\n\n    ignore_parameter_values_on_older_children : str (optional)\n        If set to ``OpenMLFlow.upload_date``, ignores parameters in a child\n        flow if it's upload date predates the upload date of the parent flow.\n\n    ignore_parameter_values : bool\n        Whether to ignore parameter values when comparing flows.\n\n    ignore_custom_name_if_none : bool\n        Whether to ignore the custom name field if either flow has `custom_name` equal to `None`.\n\n    check_description : bool\n        Whether to ignore matching of flow descriptions.\n    \"\"\"\n    if not isinstance(flow1, OpenMLFlow):\n        raise TypeError(\"Argument 1 must be of type OpenMLFlow, but is %s\" % type(flow1))\n\n    if not isinstance(flow2, OpenMLFlow):\n        raise TypeError(\"Argument 2 must be of type OpenMLFlow, but is %s\" % type(flow2))\n\n    # TODO as they are actually now saved during publish, it might be good to\n    # check for the equality of these as well.\n    generated_by_the_server = [\n        \"flow_id\",\n        \"uploader\",\n        \"version\",\n        \"upload_date\",\n        # Tags aren't directly created by the server,\n        # but the uploader has no control over them!\n        \"tags\",\n    ]\n    ignored_by_python_api = [\"binary_url\", \"binary_format\", \"binary_md5\", \"model\", \"_entity_id\"]\n\n    for key in set(flow1.__dict__.keys()).union(flow2.__dict__.keys()):\n        if key in generated_by_the_server + ignored_by_python_api:\n            continue\n        attr1 = getattr(flow1, key, None)\n        attr2 = getattr(flow2, key, None)\n        if key == \"components\":\n            if not (isinstance(attr1, Dict) and isinstance(attr2, Dict)):\n                raise TypeError(\"Cannot compare components because they are not dictionary.\")\n\n            for name in set(attr1.keys()).union(attr2.keys()):\n                if name not in attr1:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                if name not in attr2:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                assert_flows_equal(\n                    attr1[name],\n                    attr2[name],\n                    ignore_parameter_values_on_older_children,\n                    ignore_parameter_values,\n                    ignore_custom_name_if_none,\n                )\n        elif key == \"_extension\":\n            continue\n        elif check_description and key == \"description\":\n            # to ignore matching of descriptions since sklearn based flows may have\n            # altering docstrings and is not guaranteed to be consistent\n            continue\n        else:\n            if key == \"parameters\":\n                if ignore_parameter_values or ignore_parameter_values_on_older_children:\n                    params_flow_1 = set(flow1.parameters.keys())\n                    params_flow_2 = set(flow2.parameters.keys())\n                    symmetric_difference = params_flow_1 ^ params_flow_2\n                    if len(symmetric_difference) &gt; 0:\n                        raise ValueError(\n                            \"Flow %s: parameter set of flow \"\n                            \"differs from the parameters stored \"\n                            \"on the server.\" % flow1.name,\n                        )\n\n                if ignore_parameter_values_on_older_children:\n                    assert (\n                        flow1.upload_date is not None\n                    ), \"Flow1 has no upload date that allows us to compare age of children.\"\n                    upload_date_current_flow = dateutil.parser.parse(flow1.upload_date)\n                    upload_date_parent_flow = dateutil.parser.parse(\n                        ignore_parameter_values_on_older_children,\n                    )\n                    if upload_date_current_flow &lt; upload_date_parent_flow:\n                        continue\n\n                if ignore_parameter_values:\n                    # Continue needs to be done here as the first if\n                    # statement triggers in both special cases\n                    continue\n            elif (\n                key == \"custom_name\"\n                and ignore_custom_name_if_none\n                and (attr1 is None or attr2 is None)\n            ):\n                # If specified, we allow `custom_name` inequality if one flow's name is None.\n                # Helps with backwards compatibility as `custom_name` is now auto-generated, but\n                # before it used to be `None`.\n                continue\n            elif key == \"parameters_meta_info\":\n                # this value is a dictionary where each key is a parameter name, containing another\n                # dictionary with keys specifying the parameter's 'description' and 'data_type'\n                # checking parameter descriptions can be ignored since that might change\n                # data type check can also be ignored if one of them is not defined, i.e., None\n                params1 = set(flow1.parameters_meta_info)\n                params2 = set(flow2.parameters_meta_info)\n                if params1 != params2:\n                    raise ValueError(\n                        \"Parameter list in meta info for parameters differ \" \"in the two flows.\",\n                    )\n                # iterating over the parameter's meta info list\n                for param in params1:\n                    if (\n                        isinstance(flow1.parameters_meta_info[param], Dict)\n                        and isinstance(flow2.parameters_meta_info[param], Dict)\n                        and \"data_type\" in flow1.parameters_meta_info[param]\n                        and \"data_type\" in flow2.parameters_meta_info[param]\n                    ):\n                        value1 = flow1.parameters_meta_info[param][\"data_type\"]\n                        value2 = flow2.parameters_meta_info[param][\"data_type\"]\n                    else:\n                        value1 = flow1.parameters_meta_info[param]\n                        value2 = flow2.parameters_meta_info[param]\n                    if value1 is None or value2 is None:\n                        continue\n\n                    if value1 != value2:\n                        raise ValueError(\n                            f\"Flow {flow1.name}: data type for parameter {param} in {key} differ \"\n                            f\"as {value1}\\nvs\\n{value2}\",\n                        )\n                # the continue is to avoid the 'attr != attr2' check at end of function\n                continue\n\n            if attr1 != attr2:\n                raise ValueError(\n                    f\"Flow {flow1.name!s}: values for attribute '{key!s}' differ: \"\n                    f\"'{attr1!s}'\\nvs\\n'{attr2!s}'.\",\n                )\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.delete_flow","title":"<code>delete_flow(flow_id)</code>","text":"<p>Delete flow with id <code>flow_id</code> from the OpenML server.</p> <p>You can only delete flows which you uploaded and which which are not linked to runs.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>OpenML id of the flow</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def delete_flow(flow_id: int) -&gt; bool:\n    \"\"\"Delete flow with id `flow_id` from the OpenML server.\n\n    You can only delete flows which you uploaded and which\n    which are not linked to runs.\n\n    Parameters\n    ----------\n    flow_id : int\n        OpenML id of the flow\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"flow\", flow_id)\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.flow_exists","title":"<code>flow_exists(name, external_version)</code>","text":"<p>Retrieves the flow id.</p> <p>A flow is uniquely identified by name + external_version.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>string</code> <p>Name of the flow</p> required <code>external_version</code> <code>string</code> <p>Version information associated with flow.</p> required <p>Returns:</p> Name Type Description <code>flow_exist</code> <code>int or bool</code> <p>flow id iff exists, False otherwise</p> Notes <p>see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def flow_exists(name: str, external_version: str) -&gt; int | bool:\n    \"\"\"Retrieves the flow id.\n\n    A flow is uniquely identified by name + external_version.\n\n    Parameters\n    ----------\n    name : string\n        Name of the flow\n    external_version : string\n        Version information associated with flow.\n\n    Returns\n    -------\n    flow_exist : int or bool\n        flow id iff exists, False otherwise\n\n    Notes\n    -----\n    see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version\n    \"\"\"\n    if not (isinstance(name, str) and len(name) &gt; 0):\n        raise ValueError(\"Argument 'name' should be a non-empty string\")\n    if not (isinstance(name, str) and len(external_version) &gt; 0):\n        raise ValueError(\"Argument 'version' should be a non-empty string\")\n\n    xml_response = openml._api_calls._perform_api_call(\n        \"flow/exists\",\n        \"post\",\n        data={\"name\": name, \"external_version\": external_version},\n    )\n\n    result_dict = xmltodict.parse(xml_response)\n    flow_id = int(result_dict[\"oml:flow_exists\"][\"oml:id\"])\n    return flow_id if flow_id &gt; 0 else False\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.get_flow","title":"<code>get_flow(flow_id, reinstantiate=False, strict_version=True)</code>","text":"<p>Download the OpenML flow for a given flow ID.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>The OpenML flow id.</p> required <code>reinstantiate</code> <code>bool</code> <p>Whether to reinstantiate the flow to a model instance.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>flow</code> <code>OpenMLFlow</code> <p>the flow</p> Source code in <code>openml/flows/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_flow(flow_id: int, reinstantiate: bool = False, strict_version: bool = True) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Download the OpenML flow for a given flow ID.\n\n    Parameters\n    ----------\n    flow_id : int\n        The OpenML flow id.\n\n    reinstantiate: bool\n        Whether to reinstantiate the flow to a model instance.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    flow : OpenMLFlow\n        the flow\n    \"\"\"\n    flow_id = int(flow_id)\n    flow = _get_flow_description(flow_id)\n\n    if reinstantiate:\n        flow.model = flow.extension.flow_to_model(flow, strict_version=strict_version)\n        if not strict_version:\n            # check if we need to return a new flow b/c of version mismatch\n            new_flow = flow.extension.model_to_flow(flow.model)\n            if new_flow.dependencies != flow.dependencies:\n                return new_flow\n    return flow\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.get_flow_id","title":"<code>get_flow_id(model=None, name=None, exact_version=True)</code>","text":"<p>Retrieves the flow id for a model or a flow name.</p> <p>Provide either a model or a name to this function. Depending on the input, it does</p> <ul> <li><code>model</code> and <code>exact_version == True</code>: This helper function first queries for the necessary   extension. Second, it uses that extension to convert the model into a flow. Third, it   executes <code>flow_exists</code> to potentially obtain the flow id the flow is published to the   server.</li> <li><code>model</code> and <code>exact_version == False</code>: This helper function first queries for the   necessary extension. Second, it uses that extension to convert the model into a flow. Third   it calls <code>list_flows</code> and filters the returned values based on the flow name.</li> <li><code>name</code>: Ignores <code>exact_version</code> and calls <code>list_flows</code>, then filters the returned   values based on the flow name.</li> </ul> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>object</code> <p>Any model. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>name</code> <code>str</code> <p>Name of the flow. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>exact_version</code> <code>bool</code> <p>Whether to return the flow id of the exact version or all flow ids where the name of the flow matches. This is only taken into account for a model where a version number is available (requires <code>model</code> to be set).</p> <code>True</code> <p>Returns:</p> Type Description <code>(int or bool, List)</code> <p>flow id iff exists, <code>False</code> otherwise, List if <code>exact_version is False</code></p> Source code in <code>openml/flows/functions.py</code> <pre><code>def get_flow_id(\n    model: Any | None = None,\n    name: str | None = None,\n    exact_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; int | bool | list[int]:\n    \"\"\"Retrieves the flow id for a model or a flow name.\n\n    Provide either a model or a name to this function. Depending on the input, it does\n\n    * ``model`` and ``exact_version == True``: This helper function first queries for the necessary\n      extension. Second, it uses that extension to convert the model into a flow. Third, it\n      executes ``flow_exists`` to potentially obtain the flow id the flow is published to the\n      server.\n    * ``model`` and ``exact_version == False``: This helper function first queries for the\n      necessary extension. Second, it uses that extension to convert the model into a flow. Third\n      it calls ``list_flows`` and filters the returned values based on the flow name.\n    * ``name``: Ignores ``exact_version`` and calls ``list_flows``, then filters the returned\n      values based on the flow name.\n\n    Parameters\n    ----------\n    model : object\n        Any model. Must provide either ``model`` or ``name``.\n    name : str\n        Name of the flow. Must provide either ``model`` or ``name``.\n    exact_version : bool\n        Whether to return the flow id of the exact version or all flow ids where the name\n        of the flow matches. This is only taken into account for a model where a version number\n        is available (requires ``model`` to be set).\n\n    Returns\n    -------\n    int or bool, List\n        flow id iff exists, ``False`` otherwise, List if ``exact_version is False``\n    \"\"\"\n    if model is not None and name is not None:\n        raise ValueError(\"Must provide either argument `model` or argument `name`, but not both.\")\n\n    if model is not None:\n        extension = openml.extensions.get_extension_by_model(model, raise_if_no_extension=True)\n        if extension is None:\n            # This should never happen and is only here to please mypy will be gone soon once the\n            # whole function is removed\n            raise TypeError(extension)\n        flow = extension.model_to_flow(model)\n        flow_name = flow.name\n        external_version = flow.external_version\n    elif name is not None:\n        flow_name = name\n        exact_version = False\n        external_version = None\n    else:\n        raise ValueError(\n            \"Need to provide either argument `model` or argument `name`, but both are `None`.\"\n        )\n\n    if exact_version:\n        if external_version is None:\n            raise ValueError(\"exact_version should be False if model is None!\")\n        return flow_exists(name=flow_name, external_version=external_version)\n\n    flows = list_flows(output_format=\"dataframe\")\n    assert isinstance(flows, pd.DataFrame)  # Make mypy happy\n    flows = flows.query(f'name == \"{flow_name}\"')\n    return flows[\"id\"].to_list()  # type: ignore[no-any-return]\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.list_flows","title":"<code>list_flows(offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; dict\n</code></pre><pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_flows(offset: int | None, size: int | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all flows which are on OpenML. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of flows to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of flows to return</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: uploader.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>flows</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from flow_id to a dict giving a brief overview of the     respective flow.     Every flow is represented by a dictionary containing     the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> </ul> Source code in <code>openml/flows/functions.py</code> <pre><code>def list_flows(\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all flows which are on OpenML.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of flows to skip, starting from the first\n    size : int, optional\n        the maximum number of flows to return\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: uploader.\n\n    Returns\n    -------\n    flows : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from flow_id to a dict giving a brief overview of the\n            respective flow.\n            Every flow is represented by a dictionary containing\n            the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(\n        list_output_format=output_format,\n        listing_call=_list_flows,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/runs/","title":"runs","text":""},{"location":"reference/runs/#openml.runs.OpenMLRun","title":"<code>OpenMLRun</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Run: result of running a model on an OpenML dataset.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The ID of the OpenML task associated with the run.</p> required <code>flow_id</code> <code>int | None</code> <p>The ID of the OpenML flow associated with the run.</p> required <code>dataset_id</code> <code>int | None</code> <p>The ID of the OpenML dataset used for the run.</p> required <code>setup_string</code> <code>str | None</code> <p>The setup string of the run.</p> <code>None</code> <code>output_files</code> <code>dict[str, int] | None</code> <p>Specifies where each related file can be found.</p> <code>None</code> <code>setup_id</code> <code>int | None</code> <p>An integer representing the ID of the setup used for the run.</p> <code>None</code> <code>tags</code> <code>list[str] | None</code> <p>Representing the tags associated with the run.</p> <code>None</code> <code>uploader</code> <code>int | None</code> <p>User ID of the uploader.</p> <code>None</code> <code>uploader_name</code> <code>str | None</code> <p>The name of the person who uploaded the run.</p> <code>None</code> <code>evaluations</code> <code>dict | None</code> <p>Representing the evaluations of the run.</p> <code>None</code> <code>fold_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each fold.</p> <code>None</code> <code>sample_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each sample.</p> <code>None</code> <code>data_content</code> <code>list[list] | None</code> <p>The predictions generated from executing this run.</p> <code>None</code> <code>trace</code> <code>OpenMLRunTrace | None</code> <p>The trace containing information on internal model evaluations of this run.</p> <code>None</code> <code>model</code> <code>object | None</code> <p>The untrained model that was evaluated in the run.</p> <code>None</code> <code>task_type</code> <code>str | None</code> <p>The type of the OpenML task associated with the run.</p> <code>None</code> <code>task_evaluation_measure</code> <code>str | None</code> <p>The evaluation measure used for the task.</p> <code>None</code> <code>flow_name</code> <code>str | None</code> <p>The name of the OpenML flow associated with the run.</p> <code>None</code> <code>parameter_settings</code> <code>list[dict[str, Any]] | None</code> <p>Representing the parameter settings used for the run.</p> <code>None</code> <code>predictions_url</code> <code>str | None</code> <p>The URL of the predictions file.</p> <code>None</code> <code>task</code> <code>OpenMLTask | None</code> <p>An instance of the OpenMLTask class, representing the OpenML task associated with the run.</p> <code>None</code> <code>flow</code> <code>OpenMLFlow | None</code> <p>An instance of the OpenMLFlow class, representing the OpenML flow associated with the run.</p> <code>None</code> <code>run_id</code> <code>int | None</code> <p>The ID of the run.</p> <code>None</code> <code>description_text</code> <code>str | None</code> <p>Description text to add to the predictions file. If left None, is set to the time the arff file is generated.</p> <code>None</code> <code>run_details</code> <code>str | None</code> <p>Description of the run stored in the run meta-data.</p> <code>None</code> Source code in <code>openml/runs/run.py</code> <pre><code>class OpenMLRun(OpenMLBase):\n    \"\"\"OpenML Run: result of running a model on an OpenML dataset.\n\n    Parameters\n    ----------\n    task_id: int\n        The ID of the OpenML task associated with the run.\n    flow_id: int\n        The ID of the OpenML flow associated with the run.\n    dataset_id: int\n        The ID of the OpenML dataset used for the run.\n    setup_string: str\n        The setup string of the run.\n    output_files: Dict[str, int]\n        Specifies where each related file can be found.\n    setup_id: int\n        An integer representing the ID of the setup used for the run.\n    tags: List[str]\n        Representing the tags associated with the run.\n    uploader: int\n        User ID of the uploader.\n    uploader_name: str\n        The name of the person who uploaded the run.\n    evaluations: Dict\n        Representing the evaluations of the run.\n    fold_evaluations: Dict\n        The evaluations of the run for each fold.\n    sample_evaluations: Dict\n        The evaluations of the run for each sample.\n    data_content: List[List]\n        The predictions generated from executing this run.\n    trace: OpenMLRunTrace\n        The trace containing information on internal model evaluations of this run.\n    model: object\n        The untrained model that was evaluated in the run.\n    task_type: str\n        The type of the OpenML task associated with the run.\n    task_evaluation_measure: str\n        The evaluation measure used for the task.\n    flow_name: str\n        The name of the OpenML flow associated with the run.\n    parameter_settings: list[OrderedDict]\n        Representing the parameter settings used for the run.\n    predictions_url: str\n        The URL of the predictions file.\n    task: OpenMLTask\n        An instance of the OpenMLTask class, representing the OpenML task associated\n        with the run.\n    flow: OpenMLFlow\n        An instance of the OpenMLFlow class, representing the OpenML flow associated\n        with the run.\n    run_id: int\n        The ID of the run.\n    description_text: str, optional\n        Description text to add to the predictions file. If left None, is set to the\n        time the arff file is generated.\n    run_details: str, optional (default=None)\n        Description of the run stored in the run meta-data.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int,\n        flow_id: int | None,\n        dataset_id: int | None,\n        setup_string: str | None = None,\n        output_files: dict[str, int] | None = None,\n        setup_id: int | None = None,\n        tags: list[str] | None = None,\n        uploader: int | None = None,\n        uploader_name: str | None = None,\n        evaluations: dict | None = None,\n        fold_evaluations: dict | None = None,\n        sample_evaluations: dict | None = None,\n        data_content: list[list] | None = None,\n        trace: OpenMLRunTrace | None = None,\n        model: object | None = None,\n        task_type: str | None = None,\n        task_evaluation_measure: str | None = None,\n        flow_name: str | None = None,\n        parameter_settings: list[dict[str, Any]] | None = None,\n        predictions_url: str | None = None,\n        task: OpenMLTask | None = None,\n        flow: OpenMLFlow | None = None,\n        run_id: int | None = None,\n        description_text: str | None = None,\n        run_details: str | None = None,\n    ):\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.task_id = task_id\n        self.task_type = task_type\n        self.task_evaluation_measure = task_evaluation_measure\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.setup_id = setup_id\n        self.setup_string = setup_string\n        self.parameter_settings = parameter_settings\n        self.dataset_id = dataset_id\n        self.evaluations = evaluations\n        self.fold_evaluations = fold_evaluations\n        self.sample_evaluations = sample_evaluations\n        self.data_content = data_content\n        self.output_files = output_files\n        self.trace = trace\n        self.error_message = None\n        self.task = task\n        self.flow = flow\n        self.run_id = run_id\n        self.model = model\n        self.tags = tags\n        self.predictions_url = predictions_url\n        self.description_text = description_text\n        self.run_details = run_details\n        self._predictions = None\n\n    @property\n    def predictions(self) -&gt; pd.DataFrame:\n        \"\"\"Return a DataFrame with predictions for this run\"\"\"\n        if self._predictions is None:\n            if self.data_content:\n                arff_dict = self._generate_arff_dict()\n            elif self.predictions_url:\n                arff_text = openml._api_calls._download_text_file(self.predictions_url)\n                arff_dict = arff.loads(arff_text)\n            else:\n                raise RuntimeError(\"Run has no predictions.\")\n            self._predictions = pd.DataFrame(\n                arff_dict[\"data\"],\n                columns=[name for name, _ in arff_dict[\"attributes\"]],\n            )\n        return self._predictions\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the run, None if not uploaded to the server yet.\"\"\"\n        return self.run_id\n\n    def _evaluation_summary(self, metric: str) -&gt; str:\n        \"\"\"Summarizes the evaluation of a metric over all folds.\n\n        The fold scores for the metric must exist already. During run creation,\n        by default, the MAE for OpenMLRegressionTask and the accuracy for\n        OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed.\n\n        If repetition exist, we take the mean over all repetitions.\n\n        Parameters\n        ----------\n        metric: str\n            Name of an evaluation metric that was used to compute fold scores.\n\n        Returns\n        -------\n        metric_summary: str\n            A formatted string that displays the metric's evaluation summary.\n            The summary consists of the mean and std.\n        \"\"\"\n        if self.fold_evaluations is None:\n            raise ValueError(\"No fold evaluations available.\")\n        fold_score_lists = self.fold_evaluations[metric].values()\n\n        # Get the mean and std over all repetitions\n        rep_means = [np.mean(list(x.values())) for x in fold_score_lists]\n        rep_stds = [np.std(list(x.values())) for x in fold_score_lists]\n\n        return f\"{np.mean(rep_means):.4f} +- {np.mean(rep_stds):.4f}\"\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Set up fields\n        fields = {\n            \"Uploader Name\": self.uploader_name,\n            \"Metric\": self.task_evaluation_measure,\n            \"Run ID\": self.run_id,\n            \"Task ID\": self.task_id,\n            \"Task Type\": self.task_type,\n            \"Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"Flow Name\": self.flow_name,\n            \"Flow URL\": (\n                openml.flows.OpenMLFlow.url_for_id(self.flow_id)\n                if self.flow_id is not None\n                else None\n            ),\n            \"Setup ID\": self.setup_id,\n            \"Setup String\": self.setup_string,\n            \"Dataset ID\": self.dataset_id,\n            \"Dataset URL\": (\n                openml.datasets.OpenMLDataset.url_for_id(self.dataset_id)\n                if self.dataset_id is not None\n                else None\n            ),\n        }\n\n        # determines the order of the initial fields in which the information will be printed\n        order = [\"Uploader Name\", \"Uploader Profile\", \"Metric\", \"Result\"]\n\n        if self.uploader is not None:\n            fields[\"Uploader Profile\"] = f\"{openml.config.get_server_base_url()}/u/{self.uploader}\"\n        if self.run_id is not None:\n            fields[\"Run URL\"] = self.openml_url\n        if self.evaluations is not None and self.task_evaluation_measure in self.evaluations:\n            fields[\"Result\"] = self.evaluations[self.task_evaluation_measure]\n        elif self.fold_evaluations is not None:\n            # -- Add locally computed summary values if possible\n            if \"predictive_accuracy\" in self.fold_evaluations:\n                # OpenMLClassificationTask; OpenMLLearningCurveTask\n                result_field = \"Local Result - Accuracy (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"predictive_accuracy\")\n                order.append(result_field)\n            elif \"mean_absolute_error\" in self.fold_evaluations:\n                # OpenMLRegressionTask\n                result_field = \"Local Result - MAE (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"mean_absolute_error\")\n                order.append(result_field)\n\n            if \"usercpu_time_millis\" in self.fold_evaluations:\n                # Runtime should be available for most tasks types\n                rt_field = \"Local Runtime - ms (+- STD)\"\n                fields[rt_field] = self._evaluation_summary(\"usercpu_time_millis\")\n                order.append(rt_field)\n\n        # determines the remaining order\n        order += [\n            \"Run ID\",\n            \"Run URL\",\n            \"Task ID\",\n            \"Task Type\",\n            \"Task URL\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Setup ID\",\n            \"Setup String\",\n            \"Dataset ID\",\n            \"Dataset URL\",\n        ]\n        return [\n            (key, \"None\" if fields[key] is None else fields[key])  # type: ignore\n            for key in order\n            if key in fields\n        ]\n\n    @classmethod\n    def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n        \"\"\"\n        The inverse of the to_filesystem method. Instantiates an OpenMLRun\n        object based on files stored on the file system.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            are stored\n\n        expect_model : bool\n            if True, it requires the model pickle to be present, and an error\n            will be thrown if not. Otherwise, the model might or might not\n            be present.\n\n        Returns\n        -------\n        run : OpenMLRun\n            the re-instantiated run object\n        \"\"\"\n        # Avoiding cyclic imports\n        import openml.runs.functions\n\n        directory = Path(directory)\n        if not directory.is_dir():\n            raise ValueError(\"Could not find folder\")\n\n        description_path = directory / \"description.xml\"\n        predictions_path = directory / \"predictions.arff\"\n        trace_path = directory / \"trace.arff\"\n        model_path = directory / \"model.pkl\"\n\n        if not description_path.is_file():\n            raise ValueError(\"Could not find description.xml\")\n        if not predictions_path.is_file():\n            raise ValueError(\"Could not find predictions.arff\")\n        if (not model_path.is_file()) and expect_model:\n            raise ValueError(\"Could not find model.pkl\")\n\n        with description_path.open() as fht:\n            xml_string = fht.read()\n        run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n        if run.flow_id is None:\n            flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n            run.flow = flow\n            run.flow_name = flow.name\n\n        with predictions_path.open() as fht:\n            predictions = arff.load(fht)\n            run.data_content = predictions[\"data\"]\n\n        if model_path.is_file():\n            # note that it will load the model if the file exists, even if\n            # expect_model is False\n            with model_path.open(\"rb\") as fhb:\n                run.model = pickle.load(fhb)  # noqa: S301\n\n        if trace_path.is_file():\n            run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n        return run\n\n    def to_filesystem(\n        self,\n        directory: str | Path,\n        store_model: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        \"\"\"\n        The inverse of the from_filesystem method. Serializes a run\n        on the filesystem, to be uploaded later.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            will be stored. Should be empty\n\n        store_model : bool, optional (default=True)\n            if True, a model will be pickled as well. As this is the most\n            storage expensive part, it is often desirable to not store the\n            model.\n        \"\"\"\n        if self.data_content is None or self.model is None:\n            raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n        directory = Path(directory)\n        directory.mkdir(exist_ok=True, parents=True)\n\n        if any(directory.iterdir()):\n            raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n        run_xml = self._to_xml()\n        predictions_arff = arff.dumps(self._generate_arff_dict())\n\n        # It seems like typing does not allow to define the same variable multiple times\n        with (directory / \"description.xml\").open(\"w\") as fh:\n            fh.write(run_xml)\n        with (directory / \"predictions.arff\").open(\"w\") as fh:\n            fh.write(predictions_arff)\n        if store_model:\n            with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n                pickle.dump(self.model, fh_b)\n\n        if self.flow_id is None and self.flow is not None:\n            self.flow.to_filesystem(directory)\n\n        if self.trace is not None:\n            self.trace._to_filesystem(directory)\n\n    def _generate_arff_dict(self) -&gt; OrderedDict[str, Any]:\n        \"\"\"Generates the arff dictionary for uploading predictions to the\n        server.\n\n        Assumes that the run has been executed.\n\n        The order of the attributes follows the order defined by the Client API for R.\n\n        Returns\n        -------\n        arf_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains predictions and information about the run environment.\n        \"\"\"\n        if self.data_content is None:\n            raise ValueError(\"Run has not been executed.\")\n        if self.flow is None:\n            assert self.flow_id is not None, \"Run has no associated flow id!\"\n            self.flow = get_flow(self.flow_id)\n\n        if self.description_text is None:\n            self.description_text = time.strftime(\"%c\")\n        task = get_task(self.task_id)\n\n        arff_dict = OrderedDict()  # type: 'OrderedDict[str, Any]'\n        arff_dict[\"data\"] = self.data_content\n        arff_dict[\"description\"] = self.description_text\n        arff_dict[\"relation\"] = f\"openml_task_{task.task_id}_predictions\"\n\n        if isinstance(task, OpenMLLearningCurveTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"]\n                    + [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                    + [\n                        (\"confidence.\" + class_labels[i], \"NUMERIC\")\n                        for i in range(len(class_labels))\n                    ]\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLClassificationTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),  # Legacy\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                prediction_confidences = [\n                    (\"confidence.\" + class_labels[i], \"NUMERIC\") for i in range(len(class_labels))\n                ]\n                prediction_and_true = [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"] + prediction_and_true + prediction_confidences\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"prediction\", \"NUMERIC\"),\n                (\"truth\", \"NUMERIC\"),\n            ]\n\n        elif isinstance(task, OpenMLClusteringTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"cluster\", \"NUMERIC\"),\n            ]\n\n        else:\n            raise NotImplementedError(\"Task type %s is not yet supported.\" % str(task.task_type))\n\n        return arff_dict\n\n    def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n        \"\"\"Calculates metric scores based on predicted values. Assumes the\n        run has been executed locally (and contains run_data). Furthermore,\n        it assumes that the 'correct' or 'truth' attribute is specified in\n        the arff (which is an optional field, but always the case for\n        openml-python runs)\n\n        Parameters\n        ----------\n        sklearn_fn : function\n            a function pointer to a sklearn function that\n            accepts ``y_true``, ``y_pred`` and ``**kwargs``\n        kwargs : dict\n            kwargs for the function\n\n        Returns\n        -------\n        scores : ndarray of scores of length num_folds * num_repeats\n            metric results\n        \"\"\"\n        kwargs = kwargs if kwargs else {}\n        if self.data_content is not None and self.task_id is not None:\n            predictions_arff = self._generate_arff_dict()\n        elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n            predictions_file_url = openml._api_calls._file_id_to_url(\n                self.output_files[\"predictions\"],\n                \"predictions.arff\",\n            )\n            response = openml._api_calls._download_text_file(predictions_file_url)\n            predictions_arff = arff.loads(response)\n            # TODO: make this a stream reader\n        else:\n            raise ValueError(\n                \"Run should have been locally executed or \" \"contain outputfile reference.\",\n            )\n\n        # Need to know more about the task to compute scores correctly\n        task = get_task(self.task_id)\n\n        attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n        if (\n            task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n            and \"correct\" not in attribute_names\n        ):\n            raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n        if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n            raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n        if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n            raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n        def _attribute_list_to_dict(attribute_list):  # type: ignore\n            # convenience function: Creates a mapping to map from the name of\n            # attributes present in the arff prediction file to their index.\n            # This is necessary because the number of classes can be different\n            # for different tasks.\n            res = OrderedDict()\n            for idx in range(len(attribute_list)):\n                res[attribute_list[idx][0]] = idx\n            return res\n\n        attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n        repeat_idx = attribute_dict[\"repeat\"]\n        fold_idx = attribute_dict[\"fold\"]\n        predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n        if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n            correct_idx = attribute_dict[\"correct\"]\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            correct_idx = attribute_dict[\"truth\"]\n        has_samples = False\n        if \"sample\" in attribute_dict:\n            sample_idx = attribute_dict[\"sample\"]\n            has_samples = True\n\n        if (\n            predictions_arff[\"attributes\"][predicted_idx][1]\n            != predictions_arff[\"attributes\"][correct_idx][1]\n        ):\n            pred = predictions_arff[\"attributes\"][predicted_idx][1]\n            corr = predictions_arff[\"attributes\"][correct_idx][1]\n            raise ValueError(\n                \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n            )\n\n        # TODO: these could be cached\n        values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n        values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n        for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n            rep = line[repeat_idx]\n            fold = line[fold_idx]\n            samp = line[sample_idx] if has_samples else 0\n\n            if task.task_type_id in [\n                TaskType.SUPERVISED_CLASSIFICATION,\n                TaskType.LEARNING_CURVE,\n            ]:\n                prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                    line[predicted_idx],\n                )\n                correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n            elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n                prediction = line[predicted_idx]\n                correct = line[correct_idx]\n            if rep not in values_predict:\n                values_predict[rep] = OrderedDict()\n                values_correct[rep] = OrderedDict()\n            if fold not in values_predict[rep]:\n                values_predict[rep][fold] = OrderedDict()\n                values_correct[rep][fold] = OrderedDict()\n            if samp not in values_predict[rep][fold]:\n                values_predict[rep][fold][samp] = []\n                values_correct[rep][fold][samp] = []\n\n            values_predict[rep][fold][samp].append(prediction)\n            values_correct[rep][fold][samp].append(correct)\n\n        scores = []\n        for rep in values_predict:\n            for fold in values_predict[rep]:\n                last_sample = len(values_predict[rep][fold]) - 1\n                y_pred = values_predict[rep][fold][last_sample]\n                y_true = values_correct[rep][fold][last_sample]\n                scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n        return np.array(scores)\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.run_id = int(xml_response[\"oml:upload_run\"][\"oml:run_id\"])\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Get file_elements to upload to the server.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        if self.parameter_settings is None and self.model is None:\n            raise PyOpenMLError(\n                \"OpenMLRun must contain a model or be initialized with parameter_settings.\",\n            )\n        if self.flow_id is None:\n            if self.flow is None:\n                raise PyOpenMLError(\n                    \"OpenMLRun object does not contain a flow id or reference to OpenMLFlow \"\n                    \"(these should have been added while executing the task). \",\n                )\n\n            # publish the linked Flow before publishing the run.\n            self.flow.publish()\n            self.flow_id = self.flow.flow_id\n\n        if self.parameter_settings is None:\n            if self.flow is None:\n                assert self.flow_id is not None  # for mypy\n                self.flow = openml.flows.get_flow(self.flow_id)\n            self.parameter_settings = self.flow.extension.obtain_parameter_values(\n                self.flow,\n                self.model,\n            )\n\n        file_elements = {\"description\": (\"description.xml\", self._to_xml())}\n\n        if self.error_message is None:\n            predictions = arff.dumps(self._generate_arff_dict())\n            file_elements[\"predictions\"] = (\"predictions.arff\", predictions)\n\n        if self.trace is not None:\n            trace_arff = arff.dumps(self.trace.trace_to_arff())\n            file_elements[\"trace\"] = (\"trace.arff\", trace_arff)\n        return file_elements\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: PLR0912, C901\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        description = OrderedDict()  # type: 'OrderedDict'\n        description[\"oml:run\"] = OrderedDict()\n        description[\"oml:run\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n        description[\"oml:run\"][\"oml:task_id\"] = self.task_id\n        description[\"oml:run\"][\"oml:flow_id\"] = self.flow_id\n        if self.setup_string is not None:\n            description[\"oml:run\"][\"oml:setup_string\"] = self.setup_string\n        if self.error_message is not None:\n            description[\"oml:run\"][\"oml:error_message\"] = self.error_message\n        if self.run_details is not None:\n            description[\"oml:run\"][\"oml:run_details\"] = self.run_details\n        description[\"oml:run\"][\"oml:parameter_setting\"] = self.parameter_settings\n        if self.tags is not None:\n            description[\"oml:run\"][\"oml:tag\"] = self.tags\n        if (self.fold_evaluations is not None and len(self.fold_evaluations) &gt; 0) or (\n            self.sample_evaluations is not None and len(self.sample_evaluations) &gt; 0\n        ):\n            description[\"oml:run\"][\"oml:output_data\"] = OrderedDict()\n            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"] = []\n        if self.fold_evaluations is not None:\n            for measure in self.fold_evaluations:\n                for repeat in self.fold_evaluations[measure]:\n                    for fold, value in self.fold_evaluations[measure][repeat].items():\n                        current = OrderedDict(\n                            [\n                                (\"@repeat\", str(repeat)),\n                                (\"@fold\", str(fold)),\n                                (\"oml:name\", measure),\n                                (\"oml:value\", str(value)),\n                            ],\n                        )\n                        description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(current)\n        if self.sample_evaluations is not None:\n            for measure in self.sample_evaluations:\n                for repeat in self.sample_evaluations[measure]:\n                    for fold in self.sample_evaluations[measure][repeat]:\n                        for sample, value in self.sample_evaluations[measure][repeat][fold].items():\n                            current = OrderedDict(\n                                [\n                                    (\"@repeat\", str(repeat)),\n                                    (\"@fold\", str(fold)),\n                                    (\"@sample\", str(sample)),\n                                    (\"oml:name\", measure),\n                                    (\"oml:value\", str(value)),\n                                ],\n                            )\n                            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(\n                                current,\n                            )\n        return description\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRun.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the run, None if not uploaded to the server yet.</p>"},{"location":"reference/runs/#openml.runs.OpenMLRun.predictions","title":"<code>predictions: pd.DataFrame</code>  <code>property</code>","text":"<p>Return a DataFrame with predictions for this run</p>"},{"location":"reference/runs/#openml.runs.OpenMLRun.from_filesystem","title":"<code>from_filesystem(directory, expect_model=True)</code>  <code>classmethod</code>","text":"<p>The inverse of the to_filesystem method. Instantiates an OpenMLRun object based on files stored on the file system.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results are stored</p> required <code>expect_model</code> <code>bool</code> <p>if True, it requires the model pickle to be present, and an error will be thrown if not. Otherwise, the model might or might not be present.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>the re-instantiated run object</p> Source code in <code>openml/runs/run.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n    \"\"\"\n    The inverse of the to_filesystem method. Instantiates an OpenMLRun\n    object based on files stored on the file system.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        are stored\n\n    expect_model : bool\n        if True, it requires the model pickle to be present, and an error\n        will be thrown if not. Otherwise, the model might or might not\n        be present.\n\n    Returns\n    -------\n    run : OpenMLRun\n        the re-instantiated run object\n    \"\"\"\n    # Avoiding cyclic imports\n    import openml.runs.functions\n\n    directory = Path(directory)\n    if not directory.is_dir():\n        raise ValueError(\"Could not find folder\")\n\n    description_path = directory / \"description.xml\"\n    predictions_path = directory / \"predictions.arff\"\n    trace_path = directory / \"trace.arff\"\n    model_path = directory / \"model.pkl\"\n\n    if not description_path.is_file():\n        raise ValueError(\"Could not find description.xml\")\n    if not predictions_path.is_file():\n        raise ValueError(\"Could not find predictions.arff\")\n    if (not model_path.is_file()) and expect_model:\n        raise ValueError(\"Could not find model.pkl\")\n\n    with description_path.open() as fht:\n        xml_string = fht.read()\n    run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n    if run.flow_id is None:\n        flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n        run.flow = flow\n        run.flow_name = flow.name\n\n    with predictions_path.open() as fht:\n        predictions = arff.load(fht)\n        run.data_content = predictions[\"data\"]\n\n    if model_path.is_file():\n        # note that it will load the model if the file exists, even if\n        # expect_model is False\n        with model_path.open(\"rb\") as fhb:\n            run.model = pickle.load(fhb)  # noqa: S301\n\n    if trace_path.is_file():\n        run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n    return run\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRun.get_metric_fn","title":"<code>get_metric_fn(sklearn_fn, kwargs=None)</code>","text":"<p>Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' or 'truth' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs)</p> <p>Parameters:</p> Name Type Description Default <code>sklearn_fn</code> <code>function</code> <p>a function pointer to a sklearn function that accepts <code>y_true</code>, <code>y_pred</code> and <code>**kwargs</code></p> required <code>kwargs</code> <code>dict</code> <p>kwargs for the function</p> <code>None</code> <p>Returns:</p> Name Type Description <code>scores</code> <code>ndarray of scores of length num_folds * num_repeats</code> <p>metric results</p> Source code in <code>openml/runs/run.py</code> <pre><code>def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n    \"\"\"Calculates metric scores based on predicted values. Assumes the\n    run has been executed locally (and contains run_data). Furthermore,\n    it assumes that the 'correct' or 'truth' attribute is specified in\n    the arff (which is an optional field, but always the case for\n    openml-python runs)\n\n    Parameters\n    ----------\n    sklearn_fn : function\n        a function pointer to a sklearn function that\n        accepts ``y_true``, ``y_pred`` and ``**kwargs``\n    kwargs : dict\n        kwargs for the function\n\n    Returns\n    -------\n    scores : ndarray of scores of length num_folds * num_repeats\n        metric results\n    \"\"\"\n    kwargs = kwargs if kwargs else {}\n    if self.data_content is not None and self.task_id is not None:\n        predictions_arff = self._generate_arff_dict()\n    elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n        predictions_file_url = openml._api_calls._file_id_to_url(\n            self.output_files[\"predictions\"],\n            \"predictions.arff\",\n        )\n        response = openml._api_calls._download_text_file(predictions_file_url)\n        predictions_arff = arff.loads(response)\n        # TODO: make this a stream reader\n    else:\n        raise ValueError(\n            \"Run should have been locally executed or \" \"contain outputfile reference.\",\n        )\n\n    # Need to know more about the task to compute scores correctly\n    task = get_task(self.task_id)\n\n    attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n    if (\n        task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n        and \"correct\" not in attribute_names\n    ):\n        raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n    if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n        raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n    if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n        raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n    def _attribute_list_to_dict(attribute_list):  # type: ignore\n        # convenience function: Creates a mapping to map from the name of\n        # attributes present in the arff prediction file to their index.\n        # This is necessary because the number of classes can be different\n        # for different tasks.\n        res = OrderedDict()\n        for idx in range(len(attribute_list)):\n            res[attribute_list[idx][0]] = idx\n        return res\n\n    attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n    repeat_idx = attribute_dict[\"repeat\"]\n    fold_idx = attribute_dict[\"fold\"]\n    predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n    if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n        correct_idx = attribute_dict[\"correct\"]\n    elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n        correct_idx = attribute_dict[\"truth\"]\n    has_samples = False\n    if \"sample\" in attribute_dict:\n        sample_idx = attribute_dict[\"sample\"]\n        has_samples = True\n\n    if (\n        predictions_arff[\"attributes\"][predicted_idx][1]\n        != predictions_arff[\"attributes\"][correct_idx][1]\n    ):\n        pred = predictions_arff[\"attributes\"][predicted_idx][1]\n        corr = predictions_arff[\"attributes\"][correct_idx][1]\n        raise ValueError(\n            \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n        )\n\n    # TODO: these could be cached\n    values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n    values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n    for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n        rep = line[repeat_idx]\n        fold = line[fold_idx]\n        samp = line[sample_idx] if has_samples else 0\n\n        if task.task_type_id in [\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.LEARNING_CURVE,\n        ]:\n            prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                line[predicted_idx],\n            )\n            correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            prediction = line[predicted_idx]\n            correct = line[correct_idx]\n        if rep not in values_predict:\n            values_predict[rep] = OrderedDict()\n            values_correct[rep] = OrderedDict()\n        if fold not in values_predict[rep]:\n            values_predict[rep][fold] = OrderedDict()\n            values_correct[rep][fold] = OrderedDict()\n        if samp not in values_predict[rep][fold]:\n            values_predict[rep][fold][samp] = []\n            values_correct[rep][fold][samp] = []\n\n        values_predict[rep][fold][samp].append(prediction)\n        values_correct[rep][fold][samp].append(correct)\n\n    scores = []\n    for rep in values_predict:\n        for fold in values_predict[rep]:\n            last_sample = len(values_predict[rep][fold]) - 1\n            y_pred = values_predict[rep][fold][last_sample]\n            y_true = values_correct[rep][fold][last_sample]\n            scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n    return np.array(scores)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRun.to_filesystem","title":"<code>to_filesystem(directory, store_model=True)</code>","text":"<p>The inverse of the from_filesystem method. Serializes a run on the filesystem, to be uploaded later.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results will be stored. Should be empty</p> required <code>store_model</code> <code>(bool, optional(default=True))</code> <p>if True, a model will be pickled as well. As this is the most storage expensive part, it is often desirable to not store the model.</p> <code>True</code> Source code in <code>openml/runs/run.py</code> <pre><code>def to_filesystem(\n    self,\n    directory: str | Path,\n    store_model: bool = True,  # noqa: FBT001, FBT002\n) -&gt; None:\n    \"\"\"\n    The inverse of the from_filesystem method. Serializes a run\n    on the filesystem, to be uploaded later.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        will be stored. Should be empty\n\n    store_model : bool, optional (default=True)\n        if True, a model will be pickled as well. As this is the most\n        storage expensive part, it is often desirable to not store the\n        model.\n    \"\"\"\n    if self.data_content is None or self.model is None:\n        raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n    directory = Path(directory)\n    directory.mkdir(exist_ok=True, parents=True)\n\n    if any(directory.iterdir()):\n        raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n    run_xml = self._to_xml()\n    predictions_arff = arff.dumps(self._generate_arff_dict())\n\n    # It seems like typing does not allow to define the same variable multiple times\n    with (directory / \"description.xml\").open(\"w\") as fh:\n        fh.write(run_xml)\n    with (directory / \"predictions.arff\").open(\"w\") as fh:\n        fh.write(predictions_arff)\n    if store_model:\n        with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n            pickle.dump(self.model, fh_b)\n\n    if self.flow_id is None and self.flow is not None:\n        self.flow.to_filesystem(directory)\n\n    if self.trace is not None:\n        self.trace._to_filesystem(directory)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace","title":"<code>OpenMLRunTrace</code>","text":"<p>OpenML Run Trace: parsed output from Run Trace call</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML run id.</p> required <code>trace_iterations</code> <code>dict</code> <p>Mapping from key <code>(repeat, fold, iteration)</code> to an object of OpenMLTraceIteration.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>class OpenMLRunTrace:\n    \"\"\"OpenML Run Trace: parsed output from Run Trace call\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML run id.\n\n    trace_iterations : dict\n        Mapping from key ``(repeat, fold, iteration)`` to an object of\n        OpenMLTraceIteration.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        run_id: int | None,\n        trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n    ):\n        \"\"\"Object to hold the trace content of a run.\n\n        Parameters\n        ----------\n        run_id : int\n            Id for which the trace content is to be stored.\n        trace_iterations : List[List]\n            The trace content obtained by running a flow on a task.\n        \"\"\"\n        self.run_id = run_id\n        self.trace_iterations = trace_iterations\n\n    def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n        \"\"\"\n        Returns the trace iteration that was marked as selected. In\n        case multiple are marked as selected (should not happen) the\n        first of these is returned\n\n        Parameters\n        ----------\n        fold: int\n\n        repeat: int\n\n        Returns\n        -------\n        int\n            The trace iteration from the given fold and repeat that was\n            selected as the best iteration by the search procedure\n        \"\"\"\n        for r, f, i in self.trace_iterations:\n            if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n                return i\n        raise ValueError(\n            \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n        )\n\n    @classmethod\n    def generate(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Generates an OpenMLRunTrace.\n\n        Generates the trace object from the attributes and content extracted\n        while running the underlying flow.\n\n        Parameters\n        ----------\n        attributes : list\n            List of tuples describing the arff attributes.\n\n        content : list\n            List of lists containing information about the individual tuning\n            runs.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if content is None:\n            raise ValueError(\"Trace content not available.\")\n        if attributes is None:\n            raise ValueError(\"Trace attributes not available.\")\n        if len(content) == 0:\n            raise ValueError(\"Trace content is empty.\")\n        if len(attributes) != len(content[0]):\n            raise ValueError(\n                \"Trace_attributes and trace_content not compatible:\"\n                f\" {attributes} vs {content[0]}\",\n            )\n\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not allowed when constructing a \"\n            \"trace object from run results.\",\n        )\n\n    @classmethod\n    def _from_filesystem(cls, file_path: str | Path) -&gt; OpenMLRunTrace:\n        \"\"\"\n        Logic to deserialize the trace from the filesystem.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff is stored.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        file_path = Path(file_path)\n\n        if not file_path.exists():\n            raise ValueError(\"Trace file doesn't exist\")\n\n        with file_path.open(\"r\") as fp:\n            trace_arff = arff.load(fp)\n\n        for trace_idx in range(len(trace_arff[\"data\"])):\n            # iterate over first three entrees of a trace row\n            # (fold, repeat, trace_iteration) these should be int\n            for line_idx in range(3):\n                trace_arff[\"data\"][trace_idx][line_idx] = int(\n                    trace_arff[\"data\"][trace_idx][line_idx],\n                )\n\n        return cls.trace_from_arff(trace_arff)\n\n    def _to_filesystem(self, file_path: str | Path) -&gt; None:\n        \"\"\"Serialize the trace object to the filesystem.\n\n        Serialize the trace object as an arff.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff will be stored.\n        \"\"\"\n        trace_path = Path(file_path) / \"trace.arff\"\n\n        trace_arff = arff.dumps(self.trace_to_arff())\n        with trace_path.open(\"w\") as f:\n            f.write(trace_arff)\n\n    def trace_to_arff(self) -&gt; dict[str, Any]:\n        \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n        Uses the trace object to generate an arff dictionary representation.\n\n        Returns\n        -------\n        arff_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains information about the optimization trace.\n        \"\"\"\n        if self.trace_iterations is None:\n            raise ValueError(\"trace_iterations missing from the trace object\")\n\n        # attributes that will be in trace arff\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n        trace_attributes.extend(\n            [\n                (PREFIX + parameter, \"STRING\")\n                for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n            ],\n        )\n\n        arff_dict: dict[str, Any] = {}\n        data = []\n        for trace_iteration in self.trace_iterations.values():\n            tmp_list = []\n            for _attr, _ in trace_attributes:\n                if _attr.startswith(PREFIX):\n                    attr = _attr[len(PREFIX) :]\n                    value = trace_iteration.get_parameters()[attr]\n                else:\n                    attr = _attr\n                    value = getattr(trace_iteration, attr)\n\n                if attr == \"selected\":\n                    tmp_list.append(\"true\" if value else \"false\")\n                else:\n                    tmp_list.append(value)\n            data.append(tmp_list)\n\n        arff_dict[\"attributes\"] = trace_attributes\n        arff_dict[\"data\"] = data\n        # TODO allow to pass a trace description when running a flow\n        arff_dict[\"relation\"] = \"Trace\"\n        return arff_dict\n\n    @classmethod\n    def trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from arff trace.\n\n        Creates a trace file from arff object (for example, generated by a\n        local run).\n\n        Parameters\n        ----------\n        arff_obj : dict\n            LIAC arff obj, dict containing attributes, relation, data.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        attributes = arff_obj[\"attributes\"]\n        content = arff_obj[\"data\"]\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not supported for arff serialization\",\n        )\n\n    @classmethod\n    def _trace_from_arff_struct(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n        error_message: str,\n    ) -&gt; Self:\n        \"\"\"Generate a trace dictionary from ARFF structure.\n\n        Parameters\n        ----------\n        cls : type\n            The trace object to be created.\n        attributes : list[tuple[str, str]]\n            Attribute descriptions.\n        content : list[list[int | float | str]]]\n            List of instances.\n        error_message : str\n            Error message to raise if `setup_string` is in `attributes`.\n\n        Returns\n        -------\n        OrderedDict\n            A dictionary representing the trace.\n        \"\"\"\n        trace = OrderedDict()\n        attribute_idx = {att[0]: idx for idx, att in enumerate(attributes)}\n\n        for required_attribute in REQUIRED_ATTRIBUTES:\n            if required_attribute not in attribute_idx:\n                raise ValueError(\"arff misses required attribute: %s\" % required_attribute)\n        if \"setup_string\" in attribute_idx:\n            raise ValueError(error_message)\n\n        # note that the required attributes can not be duplicated because\n        # they are not parameters\n        parameter_attributes = []\n        for attribute in attribute_idx:\n            if attribute in REQUIRED_ATTRIBUTES or attribute == \"setup_string\":\n                continue\n\n            if not attribute.startswith(PREFIX):\n                raise ValueError(\n                    f\"Encountered unknown attribute {attribute} that does not start \"\n                    f\"with prefix {PREFIX}\",\n                )\n\n            parameter_attributes.append(attribute)\n\n        for itt in content:\n            repeat = int(itt[attribute_idx[\"repeat\"]])\n            fold = int(itt[attribute_idx[\"fold\"]])\n            iteration = int(itt[attribute_idx[\"iteration\"]])\n            evaluation = float(itt[attribute_idx[\"evaluation\"]])\n            selected_value = itt[attribute_idx[\"selected\"]]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for selected field, '\n                    \"received: %s\" % selected_value,\n                )\n\n            parameters = {\n                attribute: itt[attribute_idx[attribute]] for attribute in parameter_attributes\n            }\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=None,\n                evaluation=evaluation,\n                selected=selected,\n                parameters=parameters,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(None, trace)\n\n    @classmethod\n    def trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from xml.\n\n        Creates a trace file from the xml description.\n\n        Parameters\n        ----------\n        xml : string | file-like object\n            An xml description that can be either a `string` or a file-like\n            object.\n\n        Returns\n        -------\n        run : OpenMLRunTrace\n            Object containing the run id and a dict containing the trace\n            iterations.\n        \"\"\"\n        if isinstance(xml, Path):\n            xml = str(xml.absolute())\n\n        result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n        run_id = result_dict[\"oml:run_id\"]\n        trace = OrderedDict()\n\n        if \"oml:trace_iteration\" not in result_dict:\n            raise ValueError(\"Run does not contain valid trace. \")\n        if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n            raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n        for itt in result_dict[\"oml:trace_iteration\"]:\n            repeat = int(itt[\"oml:repeat\"])\n            fold = int(itt[\"oml:fold\"])\n            iteration = int(itt[\"oml:iteration\"])\n            setup_string = json.loads(itt[\"oml:setup_string\"])\n            evaluation = float(itt[\"oml:evaluation\"])\n            selected_value = itt[\"oml:selected\"]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for '\n                    \"selected field, received: %s\" % selected_value,\n                )\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=setup_string,\n                evaluation=evaluation,\n                selected=selected,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(run_id, trace)\n\n    @classmethod\n    def merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n        \"\"\"Merge multiple traces into a single trace.\n\n        Parameters\n        ----------\n        cls : type\n            Type of the trace object to be created.\n        traces : List[OpenMLRunTrace]\n            List of traces to merge.\n\n        Returns\n        -------\n        OpenMLRunTrace\n            A trace object representing the merged traces.\n\n        Raises\n        ------\n        ValueError\n            If the parameters in the iterations of the traces being merged are not equal.\n            If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n        \"\"\"\n        merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n        previous_iteration = None\n        for trace in traces:\n            for iteration in trace:\n                key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n                assert iteration.parameters is not None\n                param_keys = iteration.parameters.keys()\n\n                if previous_iteration is not None:\n                    trace_itr = merged_trace[previous_iteration]\n\n                    assert trace_itr.parameters is not None\n                    trace_itr_keys = trace_itr.parameters.keys()\n\n                    if list(param_keys) != list(trace_itr_keys):\n                        raise ValueError(\n                            \"Cannot merge traces because the parameters are not equal: \"\n                            \"{} vs {}\".format(\n                                list(trace_itr.parameters.keys()),\n                                list(iteration.parameters.keys()),\n                            ),\n                        )\n\n                if key in merged_trace:\n                    raise ValueError(\n                        f\"Cannot merge traces because key '{key}' was encountered twice\",\n                    )\n\n                merged_trace[key] = iteration\n                previous_iteration = key\n\n        return cls(None, merged_trace)\n\n    def __repr__(self) -&gt; str:\n        return \"[Run id: {}, {} trace iterations]\".format(\n            -1 if self.run_id is None else self.run_id,\n            len(self.trace_iterations),\n        )\n\n    def __iter__(self) -&gt; Iterator[OpenMLTraceIteration]:\n        yield from self.trace_iterations.values()\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.__init__","title":"<code>__init__(run_id, trace_iterations)</code>","text":"<p>Object to hold the trace content of a run.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Id for which the trace content is to be stored.</p> required <code>trace_iterations</code> <code>List[List]</code> <p>The trace content obtained by running a flow on a task.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>def __init__(\n    self,\n    run_id: int | None,\n    trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n):\n    \"\"\"Object to hold the trace content of a run.\n\n    Parameters\n    ----------\n    run_id : int\n        Id for which the trace content is to be stored.\n    trace_iterations : List[List]\n        The trace content obtained by running a flow on a task.\n    \"\"\"\n    self.run_id = run_id\n    self.trace_iterations = trace_iterations\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.generate","title":"<code>generate(attributes, content)</code>  <code>classmethod</code>","text":"<p>Generates an OpenMLRunTrace.</p> <p>Generates the trace object from the attributes and content extracted while running the underlying flow.</p> <p>Parameters:</p> Name Type Description Default <code>attributes</code> <code>list</code> <p>List of tuples describing the arff attributes.</p> required <code>content</code> <code>list</code> <p>List of lists containing information about the individual tuning runs.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef generate(\n    cls,\n    attributes: list[tuple[str, str]],\n    content: list[list[int | float | str]],\n) -&gt; OpenMLRunTrace:\n    \"\"\"Generates an OpenMLRunTrace.\n\n    Generates the trace object from the attributes and content extracted\n    while running the underlying flow.\n\n    Parameters\n    ----------\n    attributes : list\n        List of tuples describing the arff attributes.\n\n    content : list\n        List of lists containing information about the individual tuning\n        runs.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    if content is None:\n        raise ValueError(\"Trace content not available.\")\n    if attributes is None:\n        raise ValueError(\"Trace attributes not available.\")\n    if len(content) == 0:\n        raise ValueError(\"Trace content is empty.\")\n    if len(attributes) != len(content[0]):\n        raise ValueError(\n            \"Trace_attributes and trace_content not compatible:\"\n            f\" {attributes} vs {content[0]}\",\n        )\n\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not allowed when constructing a \"\n        \"trace object from run results.\",\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.get_selected_iteration","title":"<code>get_selected_iteration(fold, repeat)</code>","text":"<p>Returns the trace iteration that was marked as selected. In case multiple are marked as selected (should not happen) the first of these is returned</p> <p>Parameters:</p> Name Type Description Default <code>fold</code> <code>int</code> required <code>repeat</code> <code>int</code> required <p>Returns:</p> Type Description <code>int</code> <p>The trace iteration from the given fold and repeat that was selected as the best iteration by the search procedure</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n    \"\"\"\n    Returns the trace iteration that was marked as selected. In\n    case multiple are marked as selected (should not happen) the\n    first of these is returned\n\n    Parameters\n    ----------\n    fold: int\n\n    repeat: int\n\n    Returns\n    -------\n    int\n        The trace iteration from the given fold and repeat that was\n        selected as the best iteration by the search procedure\n    \"\"\"\n    for r, f, i in self.trace_iterations:\n        if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n            return i\n    raise ValueError(\n        \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.merge_traces","title":"<code>merge_traces(traces)</code>  <code>classmethod</code>","text":"<p>Merge multiple traces into a single trace.</p> <p>Parameters:</p> Name Type Description Default <code>cls</code> <code>type</code> <p>Type of the trace object to be created.</p> required <code>traces</code> <code>List[OpenMLRunTrace]</code> <p>List of traces to merge.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> <p>A trace object representing the merged traces.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the parameters in the iterations of the traces being merged are not equal. If a key (repeat, fold, iteration) is encountered twice while merging the traces.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n    \"\"\"Merge multiple traces into a single trace.\n\n    Parameters\n    ----------\n    cls : type\n        Type of the trace object to be created.\n    traces : List[OpenMLRunTrace]\n        List of traces to merge.\n\n    Returns\n    -------\n    OpenMLRunTrace\n        A trace object representing the merged traces.\n\n    Raises\n    ------\n    ValueError\n        If the parameters in the iterations of the traces being merged are not equal.\n        If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n    \"\"\"\n    merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n    previous_iteration = None\n    for trace in traces:\n        for iteration in trace:\n            key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n            assert iteration.parameters is not None\n            param_keys = iteration.parameters.keys()\n\n            if previous_iteration is not None:\n                trace_itr = merged_trace[previous_iteration]\n\n                assert trace_itr.parameters is not None\n                trace_itr_keys = trace_itr.parameters.keys()\n\n                if list(param_keys) != list(trace_itr_keys):\n                    raise ValueError(\n                        \"Cannot merge traces because the parameters are not equal: \"\n                        \"{} vs {}\".format(\n                            list(trace_itr.parameters.keys()),\n                            list(iteration.parameters.keys()),\n                        ),\n                    )\n\n            if key in merged_trace:\n                raise ValueError(\n                    f\"Cannot merge traces because key '{key}' was encountered twice\",\n                )\n\n            merged_trace[key] = iteration\n            previous_iteration = key\n\n    return cls(None, merged_trace)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.trace_from_arff","title":"<code>trace_from_arff(arff_obj)</code>  <code>classmethod</code>","text":"<p>Generate trace from arff trace.</p> <p>Creates a trace file from arff object (for example, generated by a local run).</p> <p>Parameters:</p> Name Type Description Default <code>arff_obj</code> <code>dict</code> <p>LIAC arff obj, dict containing attributes, relation, data.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from arff trace.\n\n    Creates a trace file from arff object (for example, generated by a\n    local run).\n\n    Parameters\n    ----------\n    arff_obj : dict\n        LIAC arff obj, dict containing attributes, relation, data.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    attributes = arff_obj[\"attributes\"]\n    content = arff_obj[\"data\"]\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not supported for arff serialization\",\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.trace_from_xml","title":"<code>trace_from_xml(xml)</code>  <code>classmethod</code>","text":"<p>Generate trace from xml.</p> <p>Creates a trace file from the xml description.</p> <p>Parameters:</p> Name Type Description Default <code>xml</code> <code>string | file-like object</code> <p>An xml description that can be either a <code>string</code> or a file-like object.</p> required <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRunTrace</code> <p>Object containing the run id and a dict containing the trace iterations.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from xml.\n\n    Creates a trace file from the xml description.\n\n    Parameters\n    ----------\n    xml : string | file-like object\n        An xml description that can be either a `string` or a file-like\n        object.\n\n    Returns\n    -------\n    run : OpenMLRunTrace\n        Object containing the run id and a dict containing the trace\n        iterations.\n    \"\"\"\n    if isinstance(xml, Path):\n        xml = str(xml.absolute())\n\n    result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n    run_id = result_dict[\"oml:run_id\"]\n    trace = OrderedDict()\n\n    if \"oml:trace_iteration\" not in result_dict:\n        raise ValueError(\"Run does not contain valid trace. \")\n    if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n        raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n    for itt in result_dict[\"oml:trace_iteration\"]:\n        repeat = int(itt[\"oml:repeat\"])\n        fold = int(itt[\"oml:fold\"])\n        iteration = int(itt[\"oml:iteration\"])\n        setup_string = json.loads(itt[\"oml:setup_string\"])\n        evaluation = float(itt[\"oml:evaluation\"])\n        selected_value = itt[\"oml:selected\"]\n        if selected_value == \"true\":\n            selected = True\n        elif selected_value == \"false\":\n            selected = False\n        else:\n            raise ValueError(\n                'expected {\"true\", \"false\"} value for '\n                \"selected field, received: %s\" % selected_value,\n            )\n\n        current = OpenMLTraceIteration(\n            repeat=repeat,\n            fold=fold,\n            iteration=iteration,\n            setup_string=setup_string,\n            evaluation=evaluation,\n            selected=selected,\n        )\n        trace[(repeat, fold, iteration)] = current\n\n    return cls(run_id, trace)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.trace_to_arff","title":"<code>trace_to_arff()</code>","text":"<p>Generate the arff dictionary for uploading predictions to the server.</p> <p>Uses the trace object to generate an arff dictionary representation.</p> <p>Returns:</p> Name Type Description <code>arff_dict</code> <code>dict</code> <p>Dictionary representation of the ARFF file that will be uploaded. Contains information about the optimization trace.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def trace_to_arff(self) -&gt; dict[str, Any]:\n    \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n    Uses the trace object to generate an arff dictionary representation.\n\n    Returns\n    -------\n    arff_dict : dict\n        Dictionary representation of the ARFF file that will be uploaded.\n        Contains information about the optimization trace.\n    \"\"\"\n    if self.trace_iterations is None:\n        raise ValueError(\"trace_iterations missing from the trace object\")\n\n    # attributes that will be in trace arff\n    trace_attributes = [\n        (\"repeat\", \"NUMERIC\"),\n        (\"fold\", \"NUMERIC\"),\n        (\"iteration\", \"NUMERIC\"),\n        (\"evaluation\", \"NUMERIC\"),\n        (\"selected\", [\"true\", \"false\"]),\n    ]\n    trace_attributes.extend(\n        [\n            (PREFIX + parameter, \"STRING\")\n            for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n        ],\n    )\n\n    arff_dict: dict[str, Any] = {}\n    data = []\n    for trace_iteration in self.trace_iterations.values():\n        tmp_list = []\n        for _attr, _ in trace_attributes:\n            if _attr.startswith(PREFIX):\n                attr = _attr[len(PREFIX) :]\n                value = trace_iteration.get_parameters()[attr]\n            else:\n                attr = _attr\n                value = getattr(trace_iteration, attr)\n\n            if attr == \"selected\":\n                tmp_list.append(\"true\" if value else \"false\")\n            else:\n                tmp_list.append(value)\n        data.append(tmp_list)\n\n    arff_dict[\"attributes\"] = trace_attributes\n    arff_dict[\"data\"] = data\n    # TODO allow to pass a trace description when running a flow\n    arff_dict[\"relation\"] = \"Trace\"\n    return arff_dict\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLTraceIteration","title":"<code>OpenMLTraceIteration</code>  <code>dataclass</code>","text":"<p>OpenML Trace Iteration: parsed output from Run Trace call Exactly one of <code>setup_string</code> or <code>parameters</code> must be provided.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>repeat number (in case of no repeats: 0)</p> required <code>fold</code> <code>int</code> <p>fold number (in case of no folds: 0)</p> required <code>iteration</code> <code>int</code> <p>iteration number of optimization procedure</p> required <code>setup_string</code> <code>str</code> <p>json string representing the parameters If not provided, <code>parameters</code> should be set.</p> <code>None</code> <code>evaluation</code> <code>double</code> <p>The evaluation that was awarded to this trace iteration. Measure is defined by the task</p> required <code>selected</code> <code>bool</code> <p>Whether this was the best of all iterations, and hence selected for making predictions. Per fold/repeat there should be only one iteration selected</p> required <code>parameters</code> <code>OrderedDict</code> <p>Dictionary specifying parameter names and their values. If not provided, <code>setup_string</code> should be set.</p> <code>None</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@dataclass\nclass OpenMLTraceIteration:\n    \"\"\"\n    OpenML Trace Iteration: parsed output from Run Trace call\n    Exactly one of `setup_string` or `parameters` must be provided.\n\n    Parameters\n    ----------\n    repeat : int\n        repeat number (in case of no repeats: 0)\n\n    fold : int\n        fold number (in case of no folds: 0)\n\n    iteration : int\n        iteration number of optimization procedure\n\n    setup_string : str, optional\n        json string representing the parameters\n        If not provided, ``parameters`` should be set.\n\n    evaluation : double\n        The evaluation that was awarded to this trace iteration.\n        Measure is defined by the task\n\n    selected : bool\n        Whether this was the best of all iterations, and hence\n        selected for making predictions. Per fold/repeat there\n        should be only one iteration selected\n\n    parameters : OrderedDict, optional\n        Dictionary specifying parameter names and their values.\n        If not provided, ``setup_string`` should be set.\n    \"\"\"\n\n    repeat: int\n    fold: int\n    iteration: int\n\n    evaluation: float\n    selected: bool\n\n    setup_string: dict[str, str] | None = None\n    parameters: dict[str, str | int | float] | None = None\n\n    def __post_init__(self) -&gt; None:\n        # TODO: refactor into one argument of type &lt;str | OrderedDict&gt;\n        if self.setup_string and self.parameters:\n            raise ValueError(\n                \"Can only be instantiated with either `setup_string` or `parameters` argument.\",\n            )\n\n        if not (self.setup_string or self.parameters):\n            raise ValueError(\n                \"Either `setup_string` or `parameters` needs to be passed as argument.\",\n            )\n\n        if self.parameters is not None and not isinstance(self.parameters, dict):\n            raise TypeError(\n                \"argument parameters is not an instance of OrderedDict, but %s\"\n                % str(type(self.parameters)),\n            )\n\n    def get_parameters(self) -&gt; dict[str, Any]:\n        \"\"\"Get the parameters of this trace iteration.\"\"\"\n        # parameters have prefix 'parameter_'\n        if self.setup_string:\n            return {\n                param[len(PREFIX) :]: json.loads(value)\n                for param, value in self.setup_string.items()\n            }\n\n        assert self.parameters is not None\n        return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLTraceIteration.get_parameters","title":"<code>get_parameters()</code>","text":"<p>Get the parameters of this trace iteration.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_parameters(self) -&gt; dict[str, Any]:\n    \"\"\"Get the parameters of this trace iteration.\"\"\"\n    # parameters have prefix 'parameter_'\n    if self.setup_string:\n        return {\n            param[len(PREFIX) :]: json.loads(value)\n            for param, value in self.setup_string.items()\n        }\n\n    assert self.parameters is not None\n    return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/runs/#openml.runs.delete_run","title":"<code>delete_run(run_id)</code>","text":"<p>Delete run with id <code>run_id</code> from the OpenML server.</p> <p>You can only delete runs which you uploaded.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML id of the run</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def delete_run(run_id: int) -&gt; bool:\n    \"\"\"Delete run with id `run_id` from the OpenML server.\n\n    You can only delete runs which you uploaded.\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML id of the run\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"run\", run_id)\n</code></pre>"},{"location":"reference/runs/#openml.runs.get_run","title":"<code>get_run(run_id, ignore_cache=False)</code>","text":"<p>Gets run corresponding to run_id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <code>ignore_cache</code> <code>bool</code> <p>Whether to ignore the cache. If <code>true</code> this will download and overwrite the run xml even if the requested run is already cached.</p> <code>False</code> <code>ignore_cache</code> <code>bool</code> <code>False</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Run corresponding to ID, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_run(run_id: int, ignore_cache: bool = False) -&gt; OpenMLRun:  # noqa: FBT002, FBT001\n    \"\"\"Gets run corresponding to run_id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    ignore_cache : bool\n        Whether to ignore the cache. If ``true`` this will download and overwrite the run xml\n        even if the requested run is already cached.\n\n    ignore_cache\n\n    Returns\n    -------\n    run : OpenMLRun\n        Run corresponding to ID, fetched from the server.\n    \"\"\"\n    run_dir = Path(openml.utils._create_cache_directory_for_id(RUNS_CACHE_DIR_NAME, run_id))\n    run_file = run_dir / \"description.xml\"\n\n    run_dir.mkdir(parents=True, exist_ok=True)\n\n    try:\n        if not ignore_cache:\n            return _get_cached_run(run_id)\n\n        raise OpenMLCacheException(message=\"dummy\")\n\n    except OpenMLCacheException:\n        run_xml = openml._api_calls._perform_api_call(\"run/%d\" % run_id, \"get\")\n        with run_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(run_xml)\n\n    return _create_run_from_xml(run_xml)\n</code></pre>"},{"location":"reference/runs/#openml.runs.get_run_trace","title":"<code>get_run_trace(run_id)</code>","text":"<p>Get the optimization trace object for a given run id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>OpenMLTrace</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_run_trace(run_id: int) -&gt; OpenMLRunTrace:\n    \"\"\"\n    Get the optimization trace object for a given run id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    Returns\n    -------\n    openml.runs.OpenMLTrace\n    \"\"\"\n    trace_xml = openml._api_calls._perform_api_call(\"run/trace/%d\" % run_id, \"get\")\n    return OpenMLRunTrace.trace_from_xml(trace_xml)\n</code></pre>"},{"location":"reference/runs/#openml.runs.get_runs","title":"<code>get_runs(run_ids)</code>","text":"<p>Gets all runs in run_ids list.</p> <p>Parameters:</p> Name Type Description Default <code>run_ids</code> <code>list of ints</code> required <p>Returns:</p> Name Type Description <code>runs</code> <code>list of OpenMLRun</code> <p>List of runs corresponding to IDs, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_runs(run_ids: list[int]) -&gt; list[OpenMLRun]:\n    \"\"\"Gets all runs in run_ids list.\n\n    Parameters\n    ----------\n    run_ids : list of ints\n\n    Returns\n    -------\n    runs : list of OpenMLRun\n        List of runs corresponding to IDs, fetched from the server.\n    \"\"\"\n    runs = []\n    for run_id in run_ids:\n        runs.append(get_run(run_id))\n    return runs\n</code></pre>"},{"location":"reference/runs/#openml.runs.initialize_model_from_run","title":"<code>initialize_model_from_run(run_id)</code>","text":"<p>Initialized a model based on a run_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_run(run_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a run_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.setup_id is not None\n    return initialize_model(run.setup_id)\n</code></pre>"},{"location":"reference/runs/#openml.runs.initialize_model_from_trace","title":"<code>initialize_model_from_trace(run_id, repeat, fold, iteration=None)</code>","text":"<p>Initialize a model based on the parameters that were set by an optimization procedure (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id. Should contain a trace file, otherwise a OpenMLServerException is raised</p> required <code>repeat</code> <code>int</code> <p>The repeat nr (column in trace file)</p> required <code>fold</code> <code>int</code> <p>The fold nr (column in trace file)</p> required <code>iteration</code> <code>int</code> <p>The iteration nr (column in trace file). If None, the best (selected) iteration will be searched (slow), according to the selection criteria implemented in OpenMLRunTrace.get_selected_iteration</p> <code>None</code> <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_trace(\n    run_id: int,\n    repeat: int,\n    fold: int,\n    iteration: int | None = None,\n) -&gt; Any:\n    \"\"\"\n    Initialize a model based on the parameters that were set\n    by an optimization procedure (i.e., using the exact same\n    parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id. Should contain a trace file,\n        otherwise a OpenMLServerException is raised\n\n    repeat : int\n        The repeat nr (column in trace file)\n\n    fold : int\n        The fold nr (column in trace file)\n\n    iteration : int\n        The iteration nr (column in trace file). If None, the\n        best (selected) iteration will be searched (slow),\n        according to the selection criteria implemented in\n        OpenMLRunTrace.get_selected_iteration\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.flow_id is not None\n\n    flow = get_flow(run.flow_id)\n    run_trace = get_run_trace(run_id)\n\n    if iteration is None:\n        iteration = run_trace.get_selected_iteration(repeat, fold)\n\n    request = (repeat, fold, iteration)\n    if request not in run_trace.trace_iterations:\n        raise ValueError(\"Combination repeat, fold, iteration not available\")\n    current = run_trace.trace_iterations[(repeat, fold, iteration)]\n\n    search_model = initialize_model_from_run(run_id)\n    return flow.extension.instantiate_model_from_hpo_class(search_model, current)\n</code></pre>"},{"location":"reference/runs/#openml.runs.list_runs","title":"<code>list_runs(offset=None, size=None, id=None, task=None, setup=None, flow=None, uploader=None, tag=None, study=None, display_errors=False, output_format='dict', **kwargs)</code>","text":"<p>List all runs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>id</code> <code>list</code> <code>None</code> <code>task</code> <code>list</code> <code>None</code> <code>setup</code> <code>list | None</code> <code>None</code> <code>flow</code> <code>list</code> <code>None</code> <code>uploader</code> <code>list</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>display_errors</code> <code>(bool, optional(default=None))</code> <p>Whether to list runs which have an error (for example a missing prediction file).</p> <code>False</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators: task_type.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict of dicts, or dataframe</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def list_runs(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    id: list | None = None,  # noqa: A002\n    task: list[int] | None = None,\n    setup: list | None = None,\n    flow: list | None = None,\n    uploader: list | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    display_errors: bool = False,  # noqa: FBT001, FBT002\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all runs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n\n    id : list, optional\n\n    task : list, optional\n\n    setup: list, optional\n\n    flow : list, optional\n\n    uploader : list, optional\n\n    tag : str, optional\n\n    study : int, optional\n\n    display_errors : bool, optional (default=None)\n        Whether to list runs which have an error (for example a missing\n        prediction file).\n\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    kwargs : dict, optional\n        Legal filter operators: task_type.\n\n    Returns\n    -------\n    dict of dicts, or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\"Invalid output format selected. Only 'dict' or 'dataframe' applicable.\")\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    # TODO(eddiebergman): Do we really need this runtime type validation?\n    if id is not None and (not isinstance(id, list)):\n        raise TypeError(\"id must be of type list.\")\n    if task is not None and (not isinstance(task, list)):\n        raise TypeError(\"task must be of type list.\")\n    if setup is not None and (not isinstance(setup, list)):\n        raise TypeError(\"setup must be of type list.\")\n    if flow is not None and (not isinstance(flow, list)):\n        raise TypeError(\"flow must be of type list.\")\n    if uploader is not None and (not isinstance(uploader, list)):\n        raise TypeError(\"uploader must be of type list.\")\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_runs,\n        offset=offset,\n        size=size,\n        id=id,\n        task=task,\n        setup=setup,\n        flow=flow,\n        uploader=uploader,\n        tag=tag,\n        study=study,\n        display_errors=display_errors,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.run_exists","title":"<code>run_exists(task_id, setup_id)</code>","text":"<p>Checks whether a task/setup combination is already present on the server.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> required <code>setup_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>    Set run ids for runs where flow setup_id was run on task_id. Empty</code> <p>set if it wasn't run yet.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_exists(task_id: int, setup_id: int) -&gt; set[int]:\n    \"\"\"Checks whether a task/setup combination is already present on the\n    server.\n\n    Parameters\n    ----------\n    task_id : int\n\n    setup_id : int\n\n    Returns\n    -------\n        Set run ids for runs where flow setup_id was run on task_id. Empty\n        set if it wasn't run yet.\n    \"\"\"\n    if setup_id &lt;= 0:\n        # openml setups are in range 1-inf\n        return set()\n\n    try:\n        result = list_runs(task=[task_id], setup=[setup_id], output_format=\"dataframe\")\n        assert isinstance(result, pd.DataFrame)  # TODO(eddiebergman): Remove once #1299\n        return set() if result.empty else set(result[\"run_id\"])\n    except OpenMLServerException as exception:\n        # error code implies no results. The run does not exist yet\n        if exception.code != ERROR_CODE:\n            raise exception\n        return set()\n</code></pre>"},{"location":"reference/runs/#openml.runs.run_flow_on_task","title":"<code>run_flow_on_task(flow, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model provided by the flow on the dataset defined by task.</p> <p>Takes the flow and repeat information into account. The Flow may optionally be published.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>A flow wraps a machine learning model together with relevant information. The model has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask</code> <p>Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_flow_on_task(  # noqa: C901, PLR0912, PLR0915, PLR0913\n    flow: OpenMLFlow,\n    task: OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT002, FBT001\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun:\n    \"\"\"Run the model provided by the flow on the dataset defined by task.\n\n    Takes the flow and repeat information into account.\n    The Flow may optionally be published.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        A flow wraps a machine learning model together with relevant information.\n        The model has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask\n        Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    \"\"\"\n    if flow_tags is not None and not isinstance(flow_tags, list):\n        raise ValueError(\"flow_tags should be a list\")\n\n    # TODO: At some point in the future do not allow for arguments in old order (changed 6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    if isinstance(flow, OpenMLTask) and isinstance(task, OpenMLFlow):\n        # We want to allow either order of argument (to avoid confusion).\n        warnings.warn(\n            \"The old argument order (Flow, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, Flow).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, flow = flow, task\n\n    if task.task_id is None:\n        raise ValueError(\"The task should be published at OpenML\")\n\n    if flow.model is None:\n        flow.model = flow.extension.flow_to_model(flow)\n\n    flow.model = flow.extension.seed_model(flow.model, seed=seed)\n\n    # We only need to sync with the server right now if we want to upload the flow,\n    # or ensure no duplicate runs exist. Otherwise it can be synced at upload time.\n    flow_id = None\n    if upload_flow or avoid_duplicate_runs:\n        flow_id = flow_exists(flow.name, flow.external_version)\n        if isinstance(flow.flow_id, int) and flow_id != flow.flow_id:\n            if flow_id is not False:\n                raise PyOpenMLError(\n                    \"Local flow_id does not match server flow_id: \"\n                    f\"'{flow.flow_id}' vs '{flow_id}'\",\n                )\n            raise PyOpenMLError(\n                \"Flow does not exist on the server, but 'flow.flow_id' is not None.\"\n            )\n        if upload_flow and flow_id is False:\n            flow.publish()\n            flow_id = flow.flow_id\n        elif flow_id:\n            flow_from_server = get_flow(flow_id)\n            _copy_server_fields(flow_from_server, flow)\n            if avoid_duplicate_runs:\n                flow_from_server.model = flow.model\n                setup_id = setup_exists(flow_from_server)\n                ids = run_exists(task.task_id, setup_id)\n                if ids:\n                    error_message = (\n                        \"One or more runs of this setup were already performed on the task.\"\n                    )\n                    raise OpenMLRunsExistError(ids, error_message)\n        else:\n            # Flow does not exist on server and we do not want to upload it.\n            # No sync with the server happens.\n            flow_id = None\n\n    dataset = task.get_dataset()\n\n    run_environment = flow.extension.get_version_information()\n    tags = [\"openml-python\", run_environment[1]]\n\n    if flow.extension.check_if_model_fitted(flow.model):\n        warnings.warn(\n            \"The model is already fitted!\"\n            \" This might cause inconsistency in comparison of results.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # execute the run\n    res = _run_task_get_arffcontent(\n        model=flow.model,\n        task=task,\n        extension=flow.extension,\n        add_local_measures=add_local_measures,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n\n    data_content, trace, fold_evaluations, sample_evaluations = res\n    fields = [*run_environment, time.strftime(\"%c\"), \"Created by run_flow_on_task\"]\n    generated_description = \"\\n\".join(fields)\n    run = OpenMLRun(\n        task_id=task.task_id,\n        flow_id=flow_id,\n        dataset_id=dataset.dataset_id,\n        model=flow.model,\n        flow_name=flow.name,\n        tags=tags,\n        trace=trace,\n        data_content=data_content,\n        flow=flow,\n        setup_string=flow.extension.create_setup_string(flow.model),\n        description_text=generated_description,\n    )\n\n    if (upload_flow or avoid_duplicate_runs) and flow.flow_id is not None:\n        # We only extract the parameter settings if a sync happened with the server.\n        # I.e. when the flow was uploaded or we found it in the avoid_duplicate check.\n        # Otherwise, we will do this at upload time.\n        run.parameter_settings = flow.extension.obtain_parameter_values(flow)\n\n    # now we need to attach the detailed evaluations\n    if task.task_type_id == TaskType.LEARNING_CURVE:\n        run.sample_evaluations = sample_evaluations\n    else:\n        run.fold_evaluations = fold_evaluations\n\n    if flow_id:\n        message = f\"Executed Task {task.task_id} with Flow id:{run.flow_id}\"\n    else:\n        message = f\"Executed Task {task.task_id} on local Flow with name {flow.name}.\"\n    config.logger.info(message)\n\n    return run\n</code></pre>"},{"location":"reference/runs/#openml.runs.run_model_on_task","title":"<code>run_model_on_task(model, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, return_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model on the dataset defined by the task.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>A model which has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask or int or str</code> <p>Task to perform or Task id. This may be a model instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>return_flow</code> <code>bool(default=False)</code> <p>If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> <code>flow</code> <code>OpenMLFlow (optional, only if `return_flow` is True).</code> <p>Flow generated from the model.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_model_on_task(  # noqa: PLR0913\n    model: Any,\n    task: int | str | OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT001, FBT002\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    return_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun | tuple[OpenMLRun, OpenMLFlow]:\n    \"\"\"Run the model on the dataset defined by the task.\n\n    Parameters\n    ----------\n    model : sklearn model\n        A model which has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask or int or str\n        Task to perform or Task id.\n        This may be a model instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    return_flow : bool (default=False)\n        If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    flow : OpenMLFlow (optional, only if `return_flow` is True).\n        Flow generated from the model.\n    \"\"\"\n    if avoid_duplicate_runs and not config.apikey:\n        warnings.warn(\n            \"avoid_duplicate_runs is set to True, but no API key is set. \"\n            \"Please set your API key in the OpenML configuration file, see\"\n            \"https://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial\"\n            \".html#authentication for more information on authentication.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # TODO: At some point in the future do not allow for arguments in old order (6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    # When removing this please also remove the method `is_estimator` from the extension\n    # interface as it is only used here (MF, 3-2019)\n    if isinstance(model, (int, str, OpenMLTask)):\n        warnings.warn(\n            \"The old argument order (task, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, task).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, model = model, task\n\n    extension = get_extension_by_model(model, raise_if_no_extension=True)\n    if extension is None:\n        # This should never happen and is only here to please mypy will be gone soon once the\n        # whole function is removed\n        raise TypeError(extension)\n\n    flow = extension.model_to_flow(model)\n\n    def get_task_and_type_conversion(_task: int | str | OpenMLTask) -&gt; OpenMLTask:\n        \"\"\"Retrieve an OpenMLTask object from either an integer or string ID,\n        or directly from an OpenMLTask object.\n\n        Parameters\n        ----------\n        _task : Union[int, str, OpenMLTask]\n            The task ID or the OpenMLTask object.\n\n        Returns\n        -------\n        OpenMLTask\n            The OpenMLTask object.\n        \"\"\"\n        if isinstance(_task, (int, str)):\n            return get_task(int(_task))  # type: ignore\n\n        return _task\n\n    task = get_task_and_type_conversion(task)\n\n    run = run_flow_on_task(\n        task=task,\n        flow=flow,\n        avoid_duplicate_runs=avoid_duplicate_runs,\n        flow_tags=flow_tags,\n        seed=seed,\n        add_local_measures=add_local_measures,\n        upload_flow=upload_flow,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n    if return_flow:\n        return run, flow\n    return run\n</code></pre>"},{"location":"reference/runs/functions/","title":"functions","text":""},{"location":"reference/runs/functions/#openml.runs.functions.__list_runs","title":"<code>__list_runs(api_call, output_format='dict')</code>","text":"<p>Helper function to parse API calls which are lists of runs</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def __list_runs(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\"] = \"dict\"\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Helper function to parse API calls which are lists of runs\"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    runs_dict = xmltodict.parse(xml_string, force_list=(\"oml:run\",))\n    # Minimalistic check if the XML is useful\n    if \"oml:runs\" not in runs_dict:\n        raise ValueError(f'Error in return XML, does not contain \"oml:runs\": {runs_dict}')\n\n    if \"@xmlns:oml\" not in runs_dict[\"oml:runs\"]:\n        raise ValueError(\n            f'Error in return XML, does not contain \"oml:runs\"/@xmlns:oml: {runs_dict}'\n        )\n\n    if runs_dict[\"oml:runs\"][\"@xmlns:oml\"] != \"http://openml.org/openml\":\n        raise ValueError(\n            \"Error in return XML, value of  \"\n            '\"oml:runs\"/@xmlns:oml is not '\n            f'\"http://openml.org/openml\": {runs_dict}',\n        )\n\n    assert isinstance(runs_dict[\"oml:runs\"][\"oml:run\"], list), type(runs_dict[\"oml:runs\"])\n\n    runs = {\n        int(r[\"oml:run_id\"]): {\n            \"run_id\": int(r[\"oml:run_id\"]),\n            \"task_id\": int(r[\"oml:task_id\"]),\n            \"setup_id\": int(r[\"oml:setup_id\"]),\n            \"flow_id\": int(r[\"oml:flow_id\"]),\n            \"uploader\": int(r[\"oml:uploader\"]),\n            \"task_type\": TaskType(int(r[\"oml:task_type_id\"])),\n            \"upload_time\": str(r[\"oml:upload_time\"]),\n            \"error_message\": str((r[\"oml:error_message\"]) or \"\"),\n        }\n        for r in runs_dict[\"oml:runs\"][\"oml:run\"]\n    }\n\n    if output_format == \"dataframe\":\n        runs = pd.DataFrame.from_dict(runs, orient=\"index\")\n\n    return runs\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.delete_run","title":"<code>delete_run(run_id)</code>","text":"<p>Delete run with id <code>run_id</code> from the OpenML server.</p> <p>You can only delete runs which you uploaded.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML id of the run</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def delete_run(run_id: int) -&gt; bool:\n    \"\"\"Delete run with id `run_id` from the OpenML server.\n\n    You can only delete runs which you uploaded.\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML id of the run\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"run\", run_id)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.format_prediction","title":"<code>format_prediction(task, repeat, fold, index, prediction, truth, sample=None, proba=None)</code>","text":"<p>Format the predictions in the specific order as required for the run results.</p> <p>Parameters:</p> Name Type Description Default <code>task</code> <code>OpenMLSupervisedTask</code> <p>Task for which to format the predictions.</p> required <code>repeat</code> <code>int</code> <p>From which repeat this predictions is made.</p> required <code>fold</code> <code>int</code> <p>From which fold this prediction is made.</p> required <code>index</code> <code>int</code> <p>For which index this prediction is made.</p> required <code>prediction</code> <code>str | int | float</code> <p>The predicted class label or value.</p> required <code>truth</code> <code>str | int | float</code> <p>The true class label or value.</p> required <code>sample</code> <code>int | None</code> <p>From which sample set this prediction is made. Required only for LearningCurve tasks.</p> <code>None</code> <code>proba</code> <code>dict[str, float] | None</code> <p>For classification tasks only. A mapping from each class label to their predicted probability. The dictionary should contain an entry for each of the <code>task.class_labels</code>. E.g.: {\"Iris-Setosa\": 0.2, \"Iris-Versicolor\": 0.7, \"Iris-Virginica\": 0.1}</p> <code>None</code> <p>Returns:</p> Type Description <code>A list with elements for the prediction results of a run.</code> <code>The returned order of the elements is (if available):</code> <p>[repeat, fold, sample, index, prediction, truth, *probabilities]</p> <code>This order follows the R Client API.</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def format_prediction(  # noqa: PLR0913\n    task: OpenMLSupervisedTask,\n    repeat: int,\n    fold: int,\n    index: int,\n    prediction: str | int | float,\n    truth: str | int | float,\n    sample: int | None = None,\n    proba: dict[str, float] | None = None,\n) -&gt; list[str | int | float]:\n    \"\"\"Format the predictions in the specific order as required for the run results.\n\n    Parameters\n    ----------\n    task: OpenMLSupervisedTask\n        Task for which to format the predictions.\n    repeat: int\n        From which repeat this predictions is made.\n    fold: int\n        From which fold this prediction is made.\n    index: int\n        For which index this prediction is made.\n    prediction: str, int or float\n        The predicted class label or value.\n    truth: str, int or float\n        The true class label or value.\n    sample: int, optional (default=None)\n        From which sample set this prediction is made.\n        Required only for LearningCurve tasks.\n    proba: Dict[str, float], optional (default=None)\n        For classification tasks only.\n        A mapping from each class label to their predicted probability.\n        The dictionary should contain an entry for each of the `task.class_labels`.\n        E.g.: {\"Iris-Setosa\": 0.2, \"Iris-Versicolor\": 0.7, \"Iris-Virginica\": 0.1}\n\n    Returns\n    -------\n    A list with elements for the prediction results of a run.\n\n    The returned order of the elements is (if available):\n        [repeat, fold, sample, index, prediction, truth, *probabilities]\n\n    This order follows the R Client API.\n    \"\"\"\n    if isinstance(task, OpenMLClassificationTask):\n        if proba is None:\n            raise ValueError(\"`proba` is required for classification task\")\n        if task.class_labels is None:\n            raise ValueError(\"The classification task must have class labels set\")\n        if not set(task.class_labels) == set(proba):\n            raise ValueError(\"Each class should have a predicted probability\")\n        if sample is None:\n            if isinstance(task, OpenMLLearningCurveTask):\n                raise ValueError(\"`sample` can not be none for LearningCurveTask\")\n\n            sample = 0\n        probabilities = [proba[c] for c in task.class_labels]\n        return [repeat, fold, sample, index, prediction, truth, *probabilities]\n\n    if isinstance(task, OpenMLRegressionTask):\n        return [repeat, fold, index, prediction, truth]\n\n    raise NotImplementedError(f\"Formatting for {type(task)} is not supported.\")\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.get_run","title":"<code>get_run(run_id, ignore_cache=False)</code>","text":"<p>Gets run corresponding to run_id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <code>ignore_cache</code> <code>bool</code> <p>Whether to ignore the cache. If <code>true</code> this will download and overwrite the run xml even if the requested run is already cached.</p> <code>False</code> <code>ignore_cache</code> <code>bool</code> <code>False</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Run corresponding to ID, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_run(run_id: int, ignore_cache: bool = False) -&gt; OpenMLRun:  # noqa: FBT002, FBT001\n    \"\"\"Gets run corresponding to run_id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    ignore_cache : bool\n        Whether to ignore the cache. If ``true`` this will download and overwrite the run xml\n        even if the requested run is already cached.\n\n    ignore_cache\n\n    Returns\n    -------\n    run : OpenMLRun\n        Run corresponding to ID, fetched from the server.\n    \"\"\"\n    run_dir = Path(openml.utils._create_cache_directory_for_id(RUNS_CACHE_DIR_NAME, run_id))\n    run_file = run_dir / \"description.xml\"\n\n    run_dir.mkdir(parents=True, exist_ok=True)\n\n    try:\n        if not ignore_cache:\n            return _get_cached_run(run_id)\n\n        raise OpenMLCacheException(message=\"dummy\")\n\n    except OpenMLCacheException:\n        run_xml = openml._api_calls._perform_api_call(\"run/%d\" % run_id, \"get\")\n        with run_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(run_xml)\n\n    return _create_run_from_xml(run_xml)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.get_run_trace","title":"<code>get_run_trace(run_id)</code>","text":"<p>Get the optimization trace object for a given run id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>OpenMLTrace</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_run_trace(run_id: int) -&gt; OpenMLRunTrace:\n    \"\"\"\n    Get the optimization trace object for a given run id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    Returns\n    -------\n    openml.runs.OpenMLTrace\n    \"\"\"\n    trace_xml = openml._api_calls._perform_api_call(\"run/trace/%d\" % run_id, \"get\")\n    return OpenMLRunTrace.trace_from_xml(trace_xml)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.get_runs","title":"<code>get_runs(run_ids)</code>","text":"<p>Gets all runs in run_ids list.</p> <p>Parameters:</p> Name Type Description Default <code>run_ids</code> <code>list of ints</code> required <p>Returns:</p> Name Type Description <code>runs</code> <code>list of OpenMLRun</code> <p>List of runs corresponding to IDs, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_runs(run_ids: list[int]) -&gt; list[OpenMLRun]:\n    \"\"\"Gets all runs in run_ids list.\n\n    Parameters\n    ----------\n    run_ids : list of ints\n\n    Returns\n    -------\n    runs : list of OpenMLRun\n        List of runs corresponding to IDs, fetched from the server.\n    \"\"\"\n    runs = []\n    for run_id in run_ids:\n        runs.append(get_run(run_id))\n    return runs\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.initialize_model_from_run","title":"<code>initialize_model_from_run(run_id)</code>","text":"<p>Initialized a model based on a run_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_run(run_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a run_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.setup_id is not None\n    return initialize_model(run.setup_id)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.initialize_model_from_trace","title":"<code>initialize_model_from_trace(run_id, repeat, fold, iteration=None)</code>","text":"<p>Initialize a model based on the parameters that were set by an optimization procedure (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id. Should contain a trace file, otherwise a OpenMLServerException is raised</p> required <code>repeat</code> <code>int</code> <p>The repeat nr (column in trace file)</p> required <code>fold</code> <code>int</code> <p>The fold nr (column in trace file)</p> required <code>iteration</code> <code>int</code> <p>The iteration nr (column in trace file). If None, the best (selected) iteration will be searched (slow), according to the selection criteria implemented in OpenMLRunTrace.get_selected_iteration</p> <code>None</code> <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_trace(\n    run_id: int,\n    repeat: int,\n    fold: int,\n    iteration: int | None = None,\n) -&gt; Any:\n    \"\"\"\n    Initialize a model based on the parameters that were set\n    by an optimization procedure (i.e., using the exact same\n    parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id. Should contain a trace file,\n        otherwise a OpenMLServerException is raised\n\n    repeat : int\n        The repeat nr (column in trace file)\n\n    fold : int\n        The fold nr (column in trace file)\n\n    iteration : int\n        The iteration nr (column in trace file). If None, the\n        best (selected) iteration will be searched (slow),\n        according to the selection criteria implemented in\n        OpenMLRunTrace.get_selected_iteration\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.flow_id is not None\n\n    flow = get_flow(run.flow_id)\n    run_trace = get_run_trace(run_id)\n\n    if iteration is None:\n        iteration = run_trace.get_selected_iteration(repeat, fold)\n\n    request = (repeat, fold, iteration)\n    if request not in run_trace.trace_iterations:\n        raise ValueError(\"Combination repeat, fold, iteration not available\")\n    current = run_trace.trace_iterations[(repeat, fold, iteration)]\n\n    search_model = initialize_model_from_run(run_id)\n    return flow.extension.instantiate_model_from_hpo_class(search_model, current)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.list_runs","title":"<code>list_runs(offset=None, size=None, id=None, task=None, setup=None, flow=None, uploader=None, tag=None, study=None, display_errors=False, output_format='dict', **kwargs)</code>","text":"<p>List all runs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>id</code> <code>list</code> <code>None</code> <code>task</code> <code>list</code> <code>None</code> <code>setup</code> <code>list | None</code> <code>None</code> <code>flow</code> <code>list</code> <code>None</code> <code>uploader</code> <code>list</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>display_errors</code> <code>(bool, optional(default=None))</code> <p>Whether to list runs which have an error (for example a missing prediction file).</p> <code>False</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators: task_type.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict of dicts, or dataframe</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def list_runs(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    id: list | None = None,  # noqa: A002\n    task: list[int] | None = None,\n    setup: list | None = None,\n    flow: list | None = None,\n    uploader: list | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    display_errors: bool = False,  # noqa: FBT001, FBT002\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all runs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n\n    id : list, optional\n\n    task : list, optional\n\n    setup: list, optional\n\n    flow : list, optional\n\n    uploader : list, optional\n\n    tag : str, optional\n\n    study : int, optional\n\n    display_errors : bool, optional (default=None)\n        Whether to list runs which have an error (for example a missing\n        prediction file).\n\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    kwargs : dict, optional\n        Legal filter operators: task_type.\n\n    Returns\n    -------\n    dict of dicts, or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\"Invalid output format selected. Only 'dict' or 'dataframe' applicable.\")\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    # TODO(eddiebergman): Do we really need this runtime type validation?\n    if id is not None and (not isinstance(id, list)):\n        raise TypeError(\"id must be of type list.\")\n    if task is not None and (not isinstance(task, list)):\n        raise TypeError(\"task must be of type list.\")\n    if setup is not None and (not isinstance(setup, list)):\n        raise TypeError(\"setup must be of type list.\")\n    if flow is not None and (not isinstance(flow, list)):\n        raise TypeError(\"flow must be of type list.\")\n    if uploader is not None and (not isinstance(uploader, list)):\n        raise TypeError(\"uploader must be of type list.\")\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_runs,\n        offset=offset,\n        size=size,\n        id=id,\n        task=task,\n        setup=setup,\n        flow=flow,\n        uploader=uploader,\n        tag=tag,\n        study=study,\n        display_errors=display_errors,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.run_exists","title":"<code>run_exists(task_id, setup_id)</code>","text":"<p>Checks whether a task/setup combination is already present on the server.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> required <code>setup_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>    Set run ids for runs where flow setup_id was run on task_id. Empty</code> <p>set if it wasn't run yet.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_exists(task_id: int, setup_id: int) -&gt; set[int]:\n    \"\"\"Checks whether a task/setup combination is already present on the\n    server.\n\n    Parameters\n    ----------\n    task_id : int\n\n    setup_id : int\n\n    Returns\n    -------\n        Set run ids for runs where flow setup_id was run on task_id. Empty\n        set if it wasn't run yet.\n    \"\"\"\n    if setup_id &lt;= 0:\n        # openml setups are in range 1-inf\n        return set()\n\n    try:\n        result = list_runs(task=[task_id], setup=[setup_id], output_format=\"dataframe\")\n        assert isinstance(result, pd.DataFrame)  # TODO(eddiebergman): Remove once #1299\n        return set() if result.empty else set(result[\"run_id\"])\n    except OpenMLServerException as exception:\n        # error code implies no results. The run does not exist yet\n        if exception.code != ERROR_CODE:\n            raise exception\n        return set()\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.run_flow_on_task","title":"<code>run_flow_on_task(flow, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model provided by the flow on the dataset defined by task.</p> <p>Takes the flow and repeat information into account. The Flow may optionally be published.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>A flow wraps a machine learning model together with relevant information. The model has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask</code> <p>Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_flow_on_task(  # noqa: C901, PLR0912, PLR0915, PLR0913\n    flow: OpenMLFlow,\n    task: OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT002, FBT001\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun:\n    \"\"\"Run the model provided by the flow on the dataset defined by task.\n\n    Takes the flow and repeat information into account.\n    The Flow may optionally be published.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        A flow wraps a machine learning model together with relevant information.\n        The model has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask\n        Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    \"\"\"\n    if flow_tags is not None and not isinstance(flow_tags, list):\n        raise ValueError(\"flow_tags should be a list\")\n\n    # TODO: At some point in the future do not allow for arguments in old order (changed 6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    if isinstance(flow, OpenMLTask) and isinstance(task, OpenMLFlow):\n        # We want to allow either order of argument (to avoid confusion).\n        warnings.warn(\n            \"The old argument order (Flow, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, Flow).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, flow = flow, task\n\n    if task.task_id is None:\n        raise ValueError(\"The task should be published at OpenML\")\n\n    if flow.model is None:\n        flow.model = flow.extension.flow_to_model(flow)\n\n    flow.model = flow.extension.seed_model(flow.model, seed=seed)\n\n    # We only need to sync with the server right now if we want to upload the flow,\n    # or ensure no duplicate runs exist. Otherwise it can be synced at upload time.\n    flow_id = None\n    if upload_flow or avoid_duplicate_runs:\n        flow_id = flow_exists(flow.name, flow.external_version)\n        if isinstance(flow.flow_id, int) and flow_id != flow.flow_id:\n            if flow_id is not False:\n                raise PyOpenMLError(\n                    \"Local flow_id does not match server flow_id: \"\n                    f\"'{flow.flow_id}' vs '{flow_id}'\",\n                )\n            raise PyOpenMLError(\n                \"Flow does not exist on the server, but 'flow.flow_id' is not None.\"\n            )\n        if upload_flow and flow_id is False:\n            flow.publish()\n            flow_id = flow.flow_id\n        elif flow_id:\n            flow_from_server = get_flow(flow_id)\n            _copy_server_fields(flow_from_server, flow)\n            if avoid_duplicate_runs:\n                flow_from_server.model = flow.model\n                setup_id = setup_exists(flow_from_server)\n                ids = run_exists(task.task_id, setup_id)\n                if ids:\n                    error_message = (\n                        \"One or more runs of this setup were already performed on the task.\"\n                    )\n                    raise OpenMLRunsExistError(ids, error_message)\n        else:\n            # Flow does not exist on server and we do not want to upload it.\n            # No sync with the server happens.\n            flow_id = None\n\n    dataset = task.get_dataset()\n\n    run_environment = flow.extension.get_version_information()\n    tags = [\"openml-python\", run_environment[1]]\n\n    if flow.extension.check_if_model_fitted(flow.model):\n        warnings.warn(\n            \"The model is already fitted!\"\n            \" This might cause inconsistency in comparison of results.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # execute the run\n    res = _run_task_get_arffcontent(\n        model=flow.model,\n        task=task,\n        extension=flow.extension,\n        add_local_measures=add_local_measures,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n\n    data_content, trace, fold_evaluations, sample_evaluations = res\n    fields = [*run_environment, time.strftime(\"%c\"), \"Created by run_flow_on_task\"]\n    generated_description = \"\\n\".join(fields)\n    run = OpenMLRun(\n        task_id=task.task_id,\n        flow_id=flow_id,\n        dataset_id=dataset.dataset_id,\n        model=flow.model,\n        flow_name=flow.name,\n        tags=tags,\n        trace=trace,\n        data_content=data_content,\n        flow=flow,\n        setup_string=flow.extension.create_setup_string(flow.model),\n        description_text=generated_description,\n    )\n\n    if (upload_flow or avoid_duplicate_runs) and flow.flow_id is not None:\n        # We only extract the parameter settings if a sync happened with the server.\n        # I.e. when the flow was uploaded or we found it in the avoid_duplicate check.\n        # Otherwise, we will do this at upload time.\n        run.parameter_settings = flow.extension.obtain_parameter_values(flow)\n\n    # now we need to attach the detailed evaluations\n    if task.task_type_id == TaskType.LEARNING_CURVE:\n        run.sample_evaluations = sample_evaluations\n    else:\n        run.fold_evaluations = fold_evaluations\n\n    if flow_id:\n        message = f\"Executed Task {task.task_id} with Flow id:{run.flow_id}\"\n    else:\n        message = f\"Executed Task {task.task_id} on local Flow with name {flow.name}.\"\n    config.logger.info(message)\n\n    return run\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.run_model_on_task","title":"<code>run_model_on_task(model, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, return_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model on the dataset defined by the task.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>A model which has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask or int or str</code> <p>Task to perform or Task id. This may be a model instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>return_flow</code> <code>bool(default=False)</code> <p>If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> <code>flow</code> <code>OpenMLFlow (optional, only if `return_flow` is True).</code> <p>Flow generated from the model.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_model_on_task(  # noqa: PLR0913\n    model: Any,\n    task: int | str | OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT001, FBT002\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    return_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun | tuple[OpenMLRun, OpenMLFlow]:\n    \"\"\"Run the model on the dataset defined by the task.\n\n    Parameters\n    ----------\n    model : sklearn model\n        A model which has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask or int or str\n        Task to perform or Task id.\n        This may be a model instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    return_flow : bool (default=False)\n        If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    flow : OpenMLFlow (optional, only if `return_flow` is True).\n        Flow generated from the model.\n    \"\"\"\n    if avoid_duplicate_runs and not config.apikey:\n        warnings.warn(\n            \"avoid_duplicate_runs is set to True, but no API key is set. \"\n            \"Please set your API key in the OpenML configuration file, see\"\n            \"https://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial\"\n            \".html#authentication for more information on authentication.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # TODO: At some point in the future do not allow for arguments in old order (6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    # When removing this please also remove the method `is_estimator` from the extension\n    # interface as it is only used here (MF, 3-2019)\n    if isinstance(model, (int, str, OpenMLTask)):\n        warnings.warn(\n            \"The old argument order (task, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, task).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, model = model, task\n\n    extension = get_extension_by_model(model, raise_if_no_extension=True)\n    if extension is None:\n        # This should never happen and is only here to please mypy will be gone soon once the\n        # whole function is removed\n        raise TypeError(extension)\n\n    flow = extension.model_to_flow(model)\n\n    def get_task_and_type_conversion(_task: int | str | OpenMLTask) -&gt; OpenMLTask:\n        \"\"\"Retrieve an OpenMLTask object from either an integer or string ID,\n        or directly from an OpenMLTask object.\n\n        Parameters\n        ----------\n        _task : Union[int, str, OpenMLTask]\n            The task ID or the OpenMLTask object.\n\n        Returns\n        -------\n        OpenMLTask\n            The OpenMLTask object.\n        \"\"\"\n        if isinstance(_task, (int, str)):\n            return get_task(int(_task))  # type: ignore\n\n        return _task\n\n    task = get_task_and_type_conversion(task)\n\n    run = run_flow_on_task(\n        task=task,\n        flow=flow,\n        avoid_duplicate_runs=avoid_duplicate_runs,\n        flow_tags=flow_tags,\n        seed=seed,\n        add_local_measures=add_local_measures,\n        upload_flow=upload_flow,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n    if return_flow:\n        return run, flow\n    return run\n</code></pre>"},{"location":"reference/runs/run/","title":"run","text":""},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun","title":"<code>OpenMLRun</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Run: result of running a model on an OpenML dataset.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The ID of the OpenML task associated with the run.</p> required <code>flow_id</code> <code>int | None</code> <p>The ID of the OpenML flow associated with the run.</p> required <code>dataset_id</code> <code>int | None</code> <p>The ID of the OpenML dataset used for the run.</p> required <code>setup_string</code> <code>str | None</code> <p>The setup string of the run.</p> <code>None</code> <code>output_files</code> <code>dict[str, int] | None</code> <p>Specifies where each related file can be found.</p> <code>None</code> <code>setup_id</code> <code>int | None</code> <p>An integer representing the ID of the setup used for the run.</p> <code>None</code> <code>tags</code> <code>list[str] | None</code> <p>Representing the tags associated with the run.</p> <code>None</code> <code>uploader</code> <code>int | None</code> <p>User ID of the uploader.</p> <code>None</code> <code>uploader_name</code> <code>str | None</code> <p>The name of the person who uploaded the run.</p> <code>None</code> <code>evaluations</code> <code>dict | None</code> <p>Representing the evaluations of the run.</p> <code>None</code> <code>fold_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each fold.</p> <code>None</code> <code>sample_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each sample.</p> <code>None</code> <code>data_content</code> <code>list[list] | None</code> <p>The predictions generated from executing this run.</p> <code>None</code> <code>trace</code> <code>OpenMLRunTrace | None</code> <p>The trace containing information on internal model evaluations of this run.</p> <code>None</code> <code>model</code> <code>object | None</code> <p>The untrained model that was evaluated in the run.</p> <code>None</code> <code>task_type</code> <code>str | None</code> <p>The type of the OpenML task associated with the run.</p> <code>None</code> <code>task_evaluation_measure</code> <code>str | None</code> <p>The evaluation measure used for the task.</p> <code>None</code> <code>flow_name</code> <code>str | None</code> <p>The name of the OpenML flow associated with the run.</p> <code>None</code> <code>parameter_settings</code> <code>list[dict[str, Any]] | None</code> <p>Representing the parameter settings used for the run.</p> <code>None</code> <code>predictions_url</code> <code>str | None</code> <p>The URL of the predictions file.</p> <code>None</code> <code>task</code> <code>OpenMLTask | None</code> <p>An instance of the OpenMLTask class, representing the OpenML task associated with the run.</p> <code>None</code> <code>flow</code> <code>OpenMLFlow | None</code> <p>An instance of the OpenMLFlow class, representing the OpenML flow associated with the run.</p> <code>None</code> <code>run_id</code> <code>int | None</code> <p>The ID of the run.</p> <code>None</code> <code>description_text</code> <code>str | None</code> <p>Description text to add to the predictions file. If left None, is set to the time the arff file is generated.</p> <code>None</code> <code>run_details</code> <code>str | None</code> <p>Description of the run stored in the run meta-data.</p> <code>None</code> Source code in <code>openml/runs/run.py</code> <pre><code>class OpenMLRun(OpenMLBase):\n    \"\"\"OpenML Run: result of running a model on an OpenML dataset.\n\n    Parameters\n    ----------\n    task_id: int\n        The ID of the OpenML task associated with the run.\n    flow_id: int\n        The ID of the OpenML flow associated with the run.\n    dataset_id: int\n        The ID of the OpenML dataset used for the run.\n    setup_string: str\n        The setup string of the run.\n    output_files: Dict[str, int]\n        Specifies where each related file can be found.\n    setup_id: int\n        An integer representing the ID of the setup used for the run.\n    tags: List[str]\n        Representing the tags associated with the run.\n    uploader: int\n        User ID of the uploader.\n    uploader_name: str\n        The name of the person who uploaded the run.\n    evaluations: Dict\n        Representing the evaluations of the run.\n    fold_evaluations: Dict\n        The evaluations of the run for each fold.\n    sample_evaluations: Dict\n        The evaluations of the run for each sample.\n    data_content: List[List]\n        The predictions generated from executing this run.\n    trace: OpenMLRunTrace\n        The trace containing information on internal model evaluations of this run.\n    model: object\n        The untrained model that was evaluated in the run.\n    task_type: str\n        The type of the OpenML task associated with the run.\n    task_evaluation_measure: str\n        The evaluation measure used for the task.\n    flow_name: str\n        The name of the OpenML flow associated with the run.\n    parameter_settings: list[OrderedDict]\n        Representing the parameter settings used for the run.\n    predictions_url: str\n        The URL of the predictions file.\n    task: OpenMLTask\n        An instance of the OpenMLTask class, representing the OpenML task associated\n        with the run.\n    flow: OpenMLFlow\n        An instance of the OpenMLFlow class, representing the OpenML flow associated\n        with the run.\n    run_id: int\n        The ID of the run.\n    description_text: str, optional\n        Description text to add to the predictions file. If left None, is set to the\n        time the arff file is generated.\n    run_details: str, optional (default=None)\n        Description of the run stored in the run meta-data.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int,\n        flow_id: int | None,\n        dataset_id: int | None,\n        setup_string: str | None = None,\n        output_files: dict[str, int] | None = None,\n        setup_id: int | None = None,\n        tags: list[str] | None = None,\n        uploader: int | None = None,\n        uploader_name: str | None = None,\n        evaluations: dict | None = None,\n        fold_evaluations: dict | None = None,\n        sample_evaluations: dict | None = None,\n        data_content: list[list] | None = None,\n        trace: OpenMLRunTrace | None = None,\n        model: object | None = None,\n        task_type: str | None = None,\n        task_evaluation_measure: str | None = None,\n        flow_name: str | None = None,\n        parameter_settings: list[dict[str, Any]] | None = None,\n        predictions_url: str | None = None,\n        task: OpenMLTask | None = None,\n        flow: OpenMLFlow | None = None,\n        run_id: int | None = None,\n        description_text: str | None = None,\n        run_details: str | None = None,\n    ):\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.task_id = task_id\n        self.task_type = task_type\n        self.task_evaluation_measure = task_evaluation_measure\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.setup_id = setup_id\n        self.setup_string = setup_string\n        self.parameter_settings = parameter_settings\n        self.dataset_id = dataset_id\n        self.evaluations = evaluations\n        self.fold_evaluations = fold_evaluations\n        self.sample_evaluations = sample_evaluations\n        self.data_content = data_content\n        self.output_files = output_files\n        self.trace = trace\n        self.error_message = None\n        self.task = task\n        self.flow = flow\n        self.run_id = run_id\n        self.model = model\n        self.tags = tags\n        self.predictions_url = predictions_url\n        self.description_text = description_text\n        self.run_details = run_details\n        self._predictions = None\n\n    @property\n    def predictions(self) -&gt; pd.DataFrame:\n        \"\"\"Return a DataFrame with predictions for this run\"\"\"\n        if self._predictions is None:\n            if self.data_content:\n                arff_dict = self._generate_arff_dict()\n            elif self.predictions_url:\n                arff_text = openml._api_calls._download_text_file(self.predictions_url)\n                arff_dict = arff.loads(arff_text)\n            else:\n                raise RuntimeError(\"Run has no predictions.\")\n            self._predictions = pd.DataFrame(\n                arff_dict[\"data\"],\n                columns=[name for name, _ in arff_dict[\"attributes\"]],\n            )\n        return self._predictions\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the run, None if not uploaded to the server yet.\"\"\"\n        return self.run_id\n\n    def _evaluation_summary(self, metric: str) -&gt; str:\n        \"\"\"Summarizes the evaluation of a metric over all folds.\n\n        The fold scores for the metric must exist already. During run creation,\n        by default, the MAE for OpenMLRegressionTask and the accuracy for\n        OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed.\n\n        If repetition exist, we take the mean over all repetitions.\n\n        Parameters\n        ----------\n        metric: str\n            Name of an evaluation metric that was used to compute fold scores.\n\n        Returns\n        -------\n        metric_summary: str\n            A formatted string that displays the metric's evaluation summary.\n            The summary consists of the mean and std.\n        \"\"\"\n        if self.fold_evaluations is None:\n            raise ValueError(\"No fold evaluations available.\")\n        fold_score_lists = self.fold_evaluations[metric].values()\n\n        # Get the mean and std over all repetitions\n        rep_means = [np.mean(list(x.values())) for x in fold_score_lists]\n        rep_stds = [np.std(list(x.values())) for x in fold_score_lists]\n\n        return f\"{np.mean(rep_means):.4f} +- {np.mean(rep_stds):.4f}\"\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Set up fields\n        fields = {\n            \"Uploader Name\": self.uploader_name,\n            \"Metric\": self.task_evaluation_measure,\n            \"Run ID\": self.run_id,\n            \"Task ID\": self.task_id,\n            \"Task Type\": self.task_type,\n            \"Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"Flow Name\": self.flow_name,\n            \"Flow URL\": (\n                openml.flows.OpenMLFlow.url_for_id(self.flow_id)\n                if self.flow_id is not None\n                else None\n            ),\n            \"Setup ID\": self.setup_id,\n            \"Setup String\": self.setup_string,\n            \"Dataset ID\": self.dataset_id,\n            \"Dataset URL\": (\n                openml.datasets.OpenMLDataset.url_for_id(self.dataset_id)\n                if self.dataset_id is not None\n                else None\n            ),\n        }\n\n        # determines the order of the initial fields in which the information will be printed\n        order = [\"Uploader Name\", \"Uploader Profile\", \"Metric\", \"Result\"]\n\n        if self.uploader is not None:\n            fields[\"Uploader Profile\"] = f\"{openml.config.get_server_base_url()}/u/{self.uploader}\"\n        if self.run_id is not None:\n            fields[\"Run URL\"] = self.openml_url\n        if self.evaluations is not None and self.task_evaluation_measure in self.evaluations:\n            fields[\"Result\"] = self.evaluations[self.task_evaluation_measure]\n        elif self.fold_evaluations is not None:\n            # -- Add locally computed summary values if possible\n            if \"predictive_accuracy\" in self.fold_evaluations:\n                # OpenMLClassificationTask; OpenMLLearningCurveTask\n                result_field = \"Local Result - Accuracy (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"predictive_accuracy\")\n                order.append(result_field)\n            elif \"mean_absolute_error\" in self.fold_evaluations:\n                # OpenMLRegressionTask\n                result_field = \"Local Result - MAE (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"mean_absolute_error\")\n                order.append(result_field)\n\n            if \"usercpu_time_millis\" in self.fold_evaluations:\n                # Runtime should be available for most tasks types\n                rt_field = \"Local Runtime - ms (+- STD)\"\n                fields[rt_field] = self._evaluation_summary(\"usercpu_time_millis\")\n                order.append(rt_field)\n\n        # determines the remaining order\n        order += [\n            \"Run ID\",\n            \"Run URL\",\n            \"Task ID\",\n            \"Task Type\",\n            \"Task URL\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Setup ID\",\n            \"Setup String\",\n            \"Dataset ID\",\n            \"Dataset URL\",\n        ]\n        return [\n            (key, \"None\" if fields[key] is None else fields[key])  # type: ignore\n            for key in order\n            if key in fields\n        ]\n\n    @classmethod\n    def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n        \"\"\"\n        The inverse of the to_filesystem method. Instantiates an OpenMLRun\n        object based on files stored on the file system.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            are stored\n\n        expect_model : bool\n            if True, it requires the model pickle to be present, and an error\n            will be thrown if not. Otherwise, the model might or might not\n            be present.\n\n        Returns\n        -------\n        run : OpenMLRun\n            the re-instantiated run object\n        \"\"\"\n        # Avoiding cyclic imports\n        import openml.runs.functions\n\n        directory = Path(directory)\n        if not directory.is_dir():\n            raise ValueError(\"Could not find folder\")\n\n        description_path = directory / \"description.xml\"\n        predictions_path = directory / \"predictions.arff\"\n        trace_path = directory / \"trace.arff\"\n        model_path = directory / \"model.pkl\"\n\n        if not description_path.is_file():\n            raise ValueError(\"Could not find description.xml\")\n        if not predictions_path.is_file():\n            raise ValueError(\"Could not find predictions.arff\")\n        if (not model_path.is_file()) and expect_model:\n            raise ValueError(\"Could not find model.pkl\")\n\n        with description_path.open() as fht:\n            xml_string = fht.read()\n        run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n        if run.flow_id is None:\n            flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n            run.flow = flow\n            run.flow_name = flow.name\n\n        with predictions_path.open() as fht:\n            predictions = arff.load(fht)\n            run.data_content = predictions[\"data\"]\n\n        if model_path.is_file():\n            # note that it will load the model if the file exists, even if\n            # expect_model is False\n            with model_path.open(\"rb\") as fhb:\n                run.model = pickle.load(fhb)  # noqa: S301\n\n        if trace_path.is_file():\n            run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n        return run\n\n    def to_filesystem(\n        self,\n        directory: str | Path,\n        store_model: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        \"\"\"\n        The inverse of the from_filesystem method. Serializes a run\n        on the filesystem, to be uploaded later.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            will be stored. Should be empty\n\n        store_model : bool, optional (default=True)\n            if True, a model will be pickled as well. As this is the most\n            storage expensive part, it is often desirable to not store the\n            model.\n        \"\"\"\n        if self.data_content is None or self.model is None:\n            raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n        directory = Path(directory)\n        directory.mkdir(exist_ok=True, parents=True)\n\n        if any(directory.iterdir()):\n            raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n        run_xml = self._to_xml()\n        predictions_arff = arff.dumps(self._generate_arff_dict())\n\n        # It seems like typing does not allow to define the same variable multiple times\n        with (directory / \"description.xml\").open(\"w\") as fh:\n            fh.write(run_xml)\n        with (directory / \"predictions.arff\").open(\"w\") as fh:\n            fh.write(predictions_arff)\n        if store_model:\n            with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n                pickle.dump(self.model, fh_b)\n\n        if self.flow_id is None and self.flow is not None:\n            self.flow.to_filesystem(directory)\n\n        if self.trace is not None:\n            self.trace._to_filesystem(directory)\n\n    def _generate_arff_dict(self) -&gt; OrderedDict[str, Any]:\n        \"\"\"Generates the arff dictionary for uploading predictions to the\n        server.\n\n        Assumes that the run has been executed.\n\n        The order of the attributes follows the order defined by the Client API for R.\n\n        Returns\n        -------\n        arf_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains predictions and information about the run environment.\n        \"\"\"\n        if self.data_content is None:\n            raise ValueError(\"Run has not been executed.\")\n        if self.flow is None:\n            assert self.flow_id is not None, \"Run has no associated flow id!\"\n            self.flow = get_flow(self.flow_id)\n\n        if self.description_text is None:\n            self.description_text = time.strftime(\"%c\")\n        task = get_task(self.task_id)\n\n        arff_dict = OrderedDict()  # type: 'OrderedDict[str, Any]'\n        arff_dict[\"data\"] = self.data_content\n        arff_dict[\"description\"] = self.description_text\n        arff_dict[\"relation\"] = f\"openml_task_{task.task_id}_predictions\"\n\n        if isinstance(task, OpenMLLearningCurveTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"]\n                    + [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                    + [\n                        (\"confidence.\" + class_labels[i], \"NUMERIC\")\n                        for i in range(len(class_labels))\n                    ]\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLClassificationTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),  # Legacy\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                prediction_confidences = [\n                    (\"confidence.\" + class_labels[i], \"NUMERIC\") for i in range(len(class_labels))\n                ]\n                prediction_and_true = [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"] + prediction_and_true + prediction_confidences\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"prediction\", \"NUMERIC\"),\n                (\"truth\", \"NUMERIC\"),\n            ]\n\n        elif isinstance(task, OpenMLClusteringTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"cluster\", \"NUMERIC\"),\n            ]\n\n        else:\n            raise NotImplementedError(\"Task type %s is not yet supported.\" % str(task.task_type))\n\n        return arff_dict\n\n    def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n        \"\"\"Calculates metric scores based on predicted values. Assumes the\n        run has been executed locally (and contains run_data). Furthermore,\n        it assumes that the 'correct' or 'truth' attribute is specified in\n        the arff (which is an optional field, but always the case for\n        openml-python runs)\n\n        Parameters\n        ----------\n        sklearn_fn : function\n            a function pointer to a sklearn function that\n            accepts ``y_true``, ``y_pred`` and ``**kwargs``\n        kwargs : dict\n            kwargs for the function\n\n        Returns\n        -------\n        scores : ndarray of scores of length num_folds * num_repeats\n            metric results\n        \"\"\"\n        kwargs = kwargs if kwargs else {}\n        if self.data_content is not None and self.task_id is not None:\n            predictions_arff = self._generate_arff_dict()\n        elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n            predictions_file_url = openml._api_calls._file_id_to_url(\n                self.output_files[\"predictions\"],\n                \"predictions.arff\",\n            )\n            response = openml._api_calls._download_text_file(predictions_file_url)\n            predictions_arff = arff.loads(response)\n            # TODO: make this a stream reader\n        else:\n            raise ValueError(\n                \"Run should have been locally executed or \" \"contain outputfile reference.\",\n            )\n\n        # Need to know more about the task to compute scores correctly\n        task = get_task(self.task_id)\n\n        attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n        if (\n            task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n            and \"correct\" not in attribute_names\n        ):\n            raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n        if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n            raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n        if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n            raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n        def _attribute_list_to_dict(attribute_list):  # type: ignore\n            # convenience function: Creates a mapping to map from the name of\n            # attributes present in the arff prediction file to their index.\n            # This is necessary because the number of classes can be different\n            # for different tasks.\n            res = OrderedDict()\n            for idx in range(len(attribute_list)):\n                res[attribute_list[idx][0]] = idx\n            return res\n\n        attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n        repeat_idx = attribute_dict[\"repeat\"]\n        fold_idx = attribute_dict[\"fold\"]\n        predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n        if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n            correct_idx = attribute_dict[\"correct\"]\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            correct_idx = attribute_dict[\"truth\"]\n        has_samples = False\n        if \"sample\" in attribute_dict:\n            sample_idx = attribute_dict[\"sample\"]\n            has_samples = True\n\n        if (\n            predictions_arff[\"attributes\"][predicted_idx][1]\n            != predictions_arff[\"attributes\"][correct_idx][1]\n        ):\n            pred = predictions_arff[\"attributes\"][predicted_idx][1]\n            corr = predictions_arff[\"attributes\"][correct_idx][1]\n            raise ValueError(\n                \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n            )\n\n        # TODO: these could be cached\n        values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n        values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n        for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n            rep = line[repeat_idx]\n            fold = line[fold_idx]\n            samp = line[sample_idx] if has_samples else 0\n\n            if task.task_type_id in [\n                TaskType.SUPERVISED_CLASSIFICATION,\n                TaskType.LEARNING_CURVE,\n            ]:\n                prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                    line[predicted_idx],\n                )\n                correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n            elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n                prediction = line[predicted_idx]\n                correct = line[correct_idx]\n            if rep not in values_predict:\n                values_predict[rep] = OrderedDict()\n                values_correct[rep] = OrderedDict()\n            if fold not in values_predict[rep]:\n                values_predict[rep][fold] = OrderedDict()\n                values_correct[rep][fold] = OrderedDict()\n            if samp not in values_predict[rep][fold]:\n                values_predict[rep][fold][samp] = []\n                values_correct[rep][fold][samp] = []\n\n            values_predict[rep][fold][samp].append(prediction)\n            values_correct[rep][fold][samp].append(correct)\n\n        scores = []\n        for rep in values_predict:\n            for fold in values_predict[rep]:\n                last_sample = len(values_predict[rep][fold]) - 1\n                y_pred = values_predict[rep][fold][last_sample]\n                y_true = values_correct[rep][fold][last_sample]\n                scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n        return np.array(scores)\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.run_id = int(xml_response[\"oml:upload_run\"][\"oml:run_id\"])\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Get file_elements to upload to the server.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        if self.parameter_settings is None and self.model is None:\n            raise PyOpenMLError(\n                \"OpenMLRun must contain a model or be initialized with parameter_settings.\",\n            )\n        if self.flow_id is None:\n            if self.flow is None:\n                raise PyOpenMLError(\n                    \"OpenMLRun object does not contain a flow id or reference to OpenMLFlow \"\n                    \"(these should have been added while executing the task). \",\n                )\n\n            # publish the linked Flow before publishing the run.\n            self.flow.publish()\n            self.flow_id = self.flow.flow_id\n\n        if self.parameter_settings is None:\n            if self.flow is None:\n                assert self.flow_id is not None  # for mypy\n                self.flow = openml.flows.get_flow(self.flow_id)\n            self.parameter_settings = self.flow.extension.obtain_parameter_values(\n                self.flow,\n                self.model,\n            )\n\n        file_elements = {\"description\": (\"description.xml\", self._to_xml())}\n\n        if self.error_message is None:\n            predictions = arff.dumps(self._generate_arff_dict())\n            file_elements[\"predictions\"] = (\"predictions.arff\", predictions)\n\n        if self.trace is not None:\n            trace_arff = arff.dumps(self.trace.trace_to_arff())\n            file_elements[\"trace\"] = (\"trace.arff\", trace_arff)\n        return file_elements\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: PLR0912, C901\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        description = OrderedDict()  # type: 'OrderedDict'\n        description[\"oml:run\"] = OrderedDict()\n        description[\"oml:run\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n        description[\"oml:run\"][\"oml:task_id\"] = self.task_id\n        description[\"oml:run\"][\"oml:flow_id\"] = self.flow_id\n        if self.setup_string is not None:\n            description[\"oml:run\"][\"oml:setup_string\"] = self.setup_string\n        if self.error_message is not None:\n            description[\"oml:run\"][\"oml:error_message\"] = self.error_message\n        if self.run_details is not None:\n            description[\"oml:run\"][\"oml:run_details\"] = self.run_details\n        description[\"oml:run\"][\"oml:parameter_setting\"] = self.parameter_settings\n        if self.tags is not None:\n            description[\"oml:run\"][\"oml:tag\"] = self.tags\n        if (self.fold_evaluations is not None and len(self.fold_evaluations) &gt; 0) or (\n            self.sample_evaluations is not None and len(self.sample_evaluations) &gt; 0\n        ):\n            description[\"oml:run\"][\"oml:output_data\"] = OrderedDict()\n            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"] = []\n        if self.fold_evaluations is not None:\n            for measure in self.fold_evaluations:\n                for repeat in self.fold_evaluations[measure]:\n                    for fold, value in self.fold_evaluations[measure][repeat].items():\n                        current = OrderedDict(\n                            [\n                                (\"@repeat\", str(repeat)),\n                                (\"@fold\", str(fold)),\n                                (\"oml:name\", measure),\n                                (\"oml:value\", str(value)),\n                            ],\n                        )\n                        description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(current)\n        if self.sample_evaluations is not None:\n            for measure in self.sample_evaluations:\n                for repeat in self.sample_evaluations[measure]:\n                    for fold in self.sample_evaluations[measure][repeat]:\n                        for sample, value in self.sample_evaluations[measure][repeat][fold].items():\n                            current = OrderedDict(\n                                [\n                                    (\"@repeat\", str(repeat)),\n                                    (\"@fold\", str(fold)),\n                                    (\"@sample\", str(sample)),\n                                    (\"oml:name\", measure),\n                                    (\"oml:value\", str(value)),\n                                ],\n                            )\n                            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(\n                                current,\n                            )\n        return description\n</code></pre>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the run, None if not uploaded to the server yet.</p>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.predictions","title":"<code>predictions: pd.DataFrame</code>  <code>property</code>","text":"<p>Return a DataFrame with predictions for this run</p>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.from_filesystem","title":"<code>from_filesystem(directory, expect_model=True)</code>  <code>classmethod</code>","text":"<p>The inverse of the to_filesystem method. Instantiates an OpenMLRun object based on files stored on the file system.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results are stored</p> required <code>expect_model</code> <code>bool</code> <p>if True, it requires the model pickle to be present, and an error will be thrown if not. Otherwise, the model might or might not be present.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>the re-instantiated run object</p> Source code in <code>openml/runs/run.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n    \"\"\"\n    The inverse of the to_filesystem method. Instantiates an OpenMLRun\n    object based on files stored on the file system.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        are stored\n\n    expect_model : bool\n        if True, it requires the model pickle to be present, and an error\n        will be thrown if not. Otherwise, the model might or might not\n        be present.\n\n    Returns\n    -------\n    run : OpenMLRun\n        the re-instantiated run object\n    \"\"\"\n    # Avoiding cyclic imports\n    import openml.runs.functions\n\n    directory = Path(directory)\n    if not directory.is_dir():\n        raise ValueError(\"Could not find folder\")\n\n    description_path = directory / \"description.xml\"\n    predictions_path = directory / \"predictions.arff\"\n    trace_path = directory / \"trace.arff\"\n    model_path = directory / \"model.pkl\"\n\n    if not description_path.is_file():\n        raise ValueError(\"Could not find description.xml\")\n    if not predictions_path.is_file():\n        raise ValueError(\"Could not find predictions.arff\")\n    if (not model_path.is_file()) and expect_model:\n        raise ValueError(\"Could not find model.pkl\")\n\n    with description_path.open() as fht:\n        xml_string = fht.read()\n    run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n    if run.flow_id is None:\n        flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n        run.flow = flow\n        run.flow_name = flow.name\n\n    with predictions_path.open() as fht:\n        predictions = arff.load(fht)\n        run.data_content = predictions[\"data\"]\n\n    if model_path.is_file():\n        # note that it will load the model if the file exists, even if\n        # expect_model is False\n        with model_path.open(\"rb\") as fhb:\n            run.model = pickle.load(fhb)  # noqa: S301\n\n    if trace_path.is_file():\n        run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n    return run\n</code></pre>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.get_metric_fn","title":"<code>get_metric_fn(sklearn_fn, kwargs=None)</code>","text":"<p>Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' or 'truth' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs)</p> <p>Parameters:</p> Name Type Description Default <code>sklearn_fn</code> <code>function</code> <p>a function pointer to a sklearn function that accepts <code>y_true</code>, <code>y_pred</code> and <code>**kwargs</code></p> required <code>kwargs</code> <code>dict</code> <p>kwargs for the function</p> <code>None</code> <p>Returns:</p> Name Type Description <code>scores</code> <code>ndarray of scores of length num_folds * num_repeats</code> <p>metric results</p> Source code in <code>openml/runs/run.py</code> <pre><code>def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n    \"\"\"Calculates metric scores based on predicted values. Assumes the\n    run has been executed locally (and contains run_data). Furthermore,\n    it assumes that the 'correct' or 'truth' attribute is specified in\n    the arff (which is an optional field, but always the case for\n    openml-python runs)\n\n    Parameters\n    ----------\n    sklearn_fn : function\n        a function pointer to a sklearn function that\n        accepts ``y_true``, ``y_pred`` and ``**kwargs``\n    kwargs : dict\n        kwargs for the function\n\n    Returns\n    -------\n    scores : ndarray of scores of length num_folds * num_repeats\n        metric results\n    \"\"\"\n    kwargs = kwargs if kwargs else {}\n    if self.data_content is not None and self.task_id is not None:\n        predictions_arff = self._generate_arff_dict()\n    elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n        predictions_file_url = openml._api_calls._file_id_to_url(\n            self.output_files[\"predictions\"],\n            \"predictions.arff\",\n        )\n        response = openml._api_calls._download_text_file(predictions_file_url)\n        predictions_arff = arff.loads(response)\n        # TODO: make this a stream reader\n    else:\n        raise ValueError(\n            \"Run should have been locally executed or \" \"contain outputfile reference.\",\n        )\n\n    # Need to know more about the task to compute scores correctly\n    task = get_task(self.task_id)\n\n    attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n    if (\n        task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n        and \"correct\" not in attribute_names\n    ):\n        raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n    if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n        raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n    if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n        raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n    def _attribute_list_to_dict(attribute_list):  # type: ignore\n        # convenience function: Creates a mapping to map from the name of\n        # attributes present in the arff prediction file to their index.\n        # This is necessary because the number of classes can be different\n        # for different tasks.\n        res = OrderedDict()\n        for idx in range(len(attribute_list)):\n            res[attribute_list[idx][0]] = idx\n        return res\n\n    attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n    repeat_idx = attribute_dict[\"repeat\"]\n    fold_idx = attribute_dict[\"fold\"]\n    predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n    if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n        correct_idx = attribute_dict[\"correct\"]\n    elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n        correct_idx = attribute_dict[\"truth\"]\n    has_samples = False\n    if \"sample\" in attribute_dict:\n        sample_idx = attribute_dict[\"sample\"]\n        has_samples = True\n\n    if (\n        predictions_arff[\"attributes\"][predicted_idx][1]\n        != predictions_arff[\"attributes\"][correct_idx][1]\n    ):\n        pred = predictions_arff[\"attributes\"][predicted_idx][1]\n        corr = predictions_arff[\"attributes\"][correct_idx][1]\n        raise ValueError(\n            \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n        )\n\n    # TODO: these could be cached\n    values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n    values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n    for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n        rep = line[repeat_idx]\n        fold = line[fold_idx]\n        samp = line[sample_idx] if has_samples else 0\n\n        if task.task_type_id in [\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.LEARNING_CURVE,\n        ]:\n            prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                line[predicted_idx],\n            )\n            correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            prediction = line[predicted_idx]\n            correct = line[correct_idx]\n        if rep not in values_predict:\n            values_predict[rep] = OrderedDict()\n            values_correct[rep] = OrderedDict()\n        if fold not in values_predict[rep]:\n            values_predict[rep][fold] = OrderedDict()\n            values_correct[rep][fold] = OrderedDict()\n        if samp not in values_predict[rep][fold]:\n            values_predict[rep][fold][samp] = []\n            values_correct[rep][fold][samp] = []\n\n        values_predict[rep][fold][samp].append(prediction)\n        values_correct[rep][fold][samp].append(correct)\n\n    scores = []\n    for rep in values_predict:\n        for fold in values_predict[rep]:\n            last_sample = len(values_predict[rep][fold]) - 1\n            y_pred = values_predict[rep][fold][last_sample]\n            y_true = values_correct[rep][fold][last_sample]\n            scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n    return np.array(scores)\n</code></pre>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.to_filesystem","title":"<code>to_filesystem(directory, store_model=True)</code>","text":"<p>The inverse of the from_filesystem method. Serializes a run on the filesystem, to be uploaded later.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results will be stored. Should be empty</p> required <code>store_model</code> <code>(bool, optional(default=True))</code> <p>if True, a model will be pickled as well. As this is the most storage expensive part, it is often desirable to not store the model.</p> <code>True</code> Source code in <code>openml/runs/run.py</code> <pre><code>def to_filesystem(\n    self,\n    directory: str | Path,\n    store_model: bool = True,  # noqa: FBT001, FBT002\n) -&gt; None:\n    \"\"\"\n    The inverse of the from_filesystem method. Serializes a run\n    on the filesystem, to be uploaded later.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        will be stored. Should be empty\n\n    store_model : bool, optional (default=True)\n        if True, a model will be pickled as well. As this is the most\n        storage expensive part, it is often desirable to not store the\n        model.\n    \"\"\"\n    if self.data_content is None or self.model is None:\n        raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n    directory = Path(directory)\n    directory.mkdir(exist_ok=True, parents=True)\n\n    if any(directory.iterdir()):\n        raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n    run_xml = self._to_xml()\n    predictions_arff = arff.dumps(self._generate_arff_dict())\n\n    # It seems like typing does not allow to define the same variable multiple times\n    with (directory / \"description.xml\").open(\"w\") as fh:\n        fh.write(run_xml)\n    with (directory / \"predictions.arff\").open(\"w\") as fh:\n        fh.write(predictions_arff)\n    if store_model:\n        with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n            pickle.dump(self.model, fh_b)\n\n    if self.flow_id is None and self.flow is not None:\n        self.flow.to_filesystem(directory)\n\n    if self.trace is not None:\n        self.trace._to_filesystem(directory)\n</code></pre>"},{"location":"reference/runs/trace/","title":"trace","text":""},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace","title":"<code>OpenMLRunTrace</code>","text":"<p>OpenML Run Trace: parsed output from Run Trace call</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML run id.</p> required <code>trace_iterations</code> <code>dict</code> <p>Mapping from key <code>(repeat, fold, iteration)</code> to an object of OpenMLTraceIteration.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>class OpenMLRunTrace:\n    \"\"\"OpenML Run Trace: parsed output from Run Trace call\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML run id.\n\n    trace_iterations : dict\n        Mapping from key ``(repeat, fold, iteration)`` to an object of\n        OpenMLTraceIteration.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        run_id: int | None,\n        trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n    ):\n        \"\"\"Object to hold the trace content of a run.\n\n        Parameters\n        ----------\n        run_id : int\n            Id for which the trace content is to be stored.\n        trace_iterations : List[List]\n            The trace content obtained by running a flow on a task.\n        \"\"\"\n        self.run_id = run_id\n        self.trace_iterations = trace_iterations\n\n    def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n        \"\"\"\n        Returns the trace iteration that was marked as selected. In\n        case multiple are marked as selected (should not happen) the\n        first of these is returned\n\n        Parameters\n        ----------\n        fold: int\n\n        repeat: int\n\n        Returns\n        -------\n        int\n            The trace iteration from the given fold and repeat that was\n            selected as the best iteration by the search procedure\n        \"\"\"\n        for r, f, i in self.trace_iterations:\n            if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n                return i\n        raise ValueError(\n            \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n        )\n\n    @classmethod\n    def generate(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Generates an OpenMLRunTrace.\n\n        Generates the trace object from the attributes and content extracted\n        while running the underlying flow.\n\n        Parameters\n        ----------\n        attributes : list\n            List of tuples describing the arff attributes.\n\n        content : list\n            List of lists containing information about the individual tuning\n            runs.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if content is None:\n            raise ValueError(\"Trace content not available.\")\n        if attributes is None:\n            raise ValueError(\"Trace attributes not available.\")\n        if len(content) == 0:\n            raise ValueError(\"Trace content is empty.\")\n        if len(attributes) != len(content[0]):\n            raise ValueError(\n                \"Trace_attributes and trace_content not compatible:\"\n                f\" {attributes} vs {content[0]}\",\n            )\n\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not allowed when constructing a \"\n            \"trace object from run results.\",\n        )\n\n    @classmethod\n    def _from_filesystem(cls, file_path: str | Path) -&gt; OpenMLRunTrace:\n        \"\"\"\n        Logic to deserialize the trace from the filesystem.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff is stored.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        file_path = Path(file_path)\n\n        if not file_path.exists():\n            raise ValueError(\"Trace file doesn't exist\")\n\n        with file_path.open(\"r\") as fp:\n            trace_arff = arff.load(fp)\n\n        for trace_idx in range(len(trace_arff[\"data\"])):\n            # iterate over first three entrees of a trace row\n            # (fold, repeat, trace_iteration) these should be int\n            for line_idx in range(3):\n                trace_arff[\"data\"][trace_idx][line_idx] = int(\n                    trace_arff[\"data\"][trace_idx][line_idx],\n                )\n\n        return cls.trace_from_arff(trace_arff)\n\n    def _to_filesystem(self, file_path: str | Path) -&gt; None:\n        \"\"\"Serialize the trace object to the filesystem.\n\n        Serialize the trace object as an arff.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff will be stored.\n        \"\"\"\n        trace_path = Path(file_path) / \"trace.arff\"\n\n        trace_arff = arff.dumps(self.trace_to_arff())\n        with trace_path.open(\"w\") as f:\n            f.write(trace_arff)\n\n    def trace_to_arff(self) -&gt; dict[str, Any]:\n        \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n        Uses the trace object to generate an arff dictionary representation.\n\n        Returns\n        -------\n        arff_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains information about the optimization trace.\n        \"\"\"\n        if self.trace_iterations is None:\n            raise ValueError(\"trace_iterations missing from the trace object\")\n\n        # attributes that will be in trace arff\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n        trace_attributes.extend(\n            [\n                (PREFIX + parameter, \"STRING\")\n                for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n            ],\n        )\n\n        arff_dict: dict[str, Any] = {}\n        data = []\n        for trace_iteration in self.trace_iterations.values():\n            tmp_list = []\n            for _attr, _ in trace_attributes:\n                if _attr.startswith(PREFIX):\n                    attr = _attr[len(PREFIX) :]\n                    value = trace_iteration.get_parameters()[attr]\n                else:\n                    attr = _attr\n                    value = getattr(trace_iteration, attr)\n\n                if attr == \"selected\":\n                    tmp_list.append(\"true\" if value else \"false\")\n                else:\n                    tmp_list.append(value)\n            data.append(tmp_list)\n\n        arff_dict[\"attributes\"] = trace_attributes\n        arff_dict[\"data\"] = data\n        # TODO allow to pass a trace description when running a flow\n        arff_dict[\"relation\"] = \"Trace\"\n        return arff_dict\n\n    @classmethod\n    def trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from arff trace.\n\n        Creates a trace file from arff object (for example, generated by a\n        local run).\n\n        Parameters\n        ----------\n        arff_obj : dict\n            LIAC arff obj, dict containing attributes, relation, data.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        attributes = arff_obj[\"attributes\"]\n        content = arff_obj[\"data\"]\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not supported for arff serialization\",\n        )\n\n    @classmethod\n    def _trace_from_arff_struct(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n        error_message: str,\n    ) -&gt; Self:\n        \"\"\"Generate a trace dictionary from ARFF structure.\n\n        Parameters\n        ----------\n        cls : type\n            The trace object to be created.\n        attributes : list[tuple[str, str]]\n            Attribute descriptions.\n        content : list[list[int | float | str]]]\n            List of instances.\n        error_message : str\n            Error message to raise if `setup_string` is in `attributes`.\n\n        Returns\n        -------\n        OrderedDict\n            A dictionary representing the trace.\n        \"\"\"\n        trace = OrderedDict()\n        attribute_idx = {att[0]: idx for idx, att in enumerate(attributes)}\n\n        for required_attribute in REQUIRED_ATTRIBUTES:\n            if required_attribute not in attribute_idx:\n                raise ValueError(\"arff misses required attribute: %s\" % required_attribute)\n        if \"setup_string\" in attribute_idx:\n            raise ValueError(error_message)\n\n        # note that the required attributes can not be duplicated because\n        # they are not parameters\n        parameter_attributes = []\n        for attribute in attribute_idx:\n            if attribute in REQUIRED_ATTRIBUTES or attribute == \"setup_string\":\n                continue\n\n            if not attribute.startswith(PREFIX):\n                raise ValueError(\n                    f\"Encountered unknown attribute {attribute} that does not start \"\n                    f\"with prefix {PREFIX}\",\n                )\n\n            parameter_attributes.append(attribute)\n\n        for itt in content:\n            repeat = int(itt[attribute_idx[\"repeat\"]])\n            fold = int(itt[attribute_idx[\"fold\"]])\n            iteration = int(itt[attribute_idx[\"iteration\"]])\n            evaluation = float(itt[attribute_idx[\"evaluation\"]])\n            selected_value = itt[attribute_idx[\"selected\"]]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for selected field, '\n                    \"received: %s\" % selected_value,\n                )\n\n            parameters = {\n                attribute: itt[attribute_idx[attribute]] for attribute in parameter_attributes\n            }\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=None,\n                evaluation=evaluation,\n                selected=selected,\n                parameters=parameters,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(None, trace)\n\n    @classmethod\n    def trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from xml.\n\n        Creates a trace file from the xml description.\n\n        Parameters\n        ----------\n        xml : string | file-like object\n            An xml description that can be either a `string` or a file-like\n            object.\n\n        Returns\n        -------\n        run : OpenMLRunTrace\n            Object containing the run id and a dict containing the trace\n            iterations.\n        \"\"\"\n        if isinstance(xml, Path):\n            xml = str(xml.absolute())\n\n        result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n        run_id = result_dict[\"oml:run_id\"]\n        trace = OrderedDict()\n\n        if \"oml:trace_iteration\" not in result_dict:\n            raise ValueError(\"Run does not contain valid trace. \")\n        if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n            raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n        for itt in result_dict[\"oml:trace_iteration\"]:\n            repeat = int(itt[\"oml:repeat\"])\n            fold = int(itt[\"oml:fold\"])\n            iteration = int(itt[\"oml:iteration\"])\n            setup_string = json.loads(itt[\"oml:setup_string\"])\n            evaluation = float(itt[\"oml:evaluation\"])\n            selected_value = itt[\"oml:selected\"]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for '\n                    \"selected field, received: %s\" % selected_value,\n                )\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=setup_string,\n                evaluation=evaluation,\n                selected=selected,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(run_id, trace)\n\n    @classmethod\n    def merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n        \"\"\"Merge multiple traces into a single trace.\n\n        Parameters\n        ----------\n        cls : type\n            Type of the trace object to be created.\n        traces : List[OpenMLRunTrace]\n            List of traces to merge.\n\n        Returns\n        -------\n        OpenMLRunTrace\n            A trace object representing the merged traces.\n\n        Raises\n        ------\n        ValueError\n            If the parameters in the iterations of the traces being merged are not equal.\n            If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n        \"\"\"\n        merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n        previous_iteration = None\n        for trace in traces:\n            for iteration in trace:\n                key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n                assert iteration.parameters is not None\n                param_keys = iteration.parameters.keys()\n\n                if previous_iteration is not None:\n                    trace_itr = merged_trace[previous_iteration]\n\n                    assert trace_itr.parameters is not None\n                    trace_itr_keys = trace_itr.parameters.keys()\n\n                    if list(param_keys) != list(trace_itr_keys):\n                        raise ValueError(\n                            \"Cannot merge traces because the parameters are not equal: \"\n                            \"{} vs {}\".format(\n                                list(trace_itr.parameters.keys()),\n                                list(iteration.parameters.keys()),\n                            ),\n                        )\n\n                if key in merged_trace:\n                    raise ValueError(\n                        f\"Cannot merge traces because key '{key}' was encountered twice\",\n                    )\n\n                merged_trace[key] = iteration\n                previous_iteration = key\n\n        return cls(None, merged_trace)\n\n    def __repr__(self) -&gt; str:\n        return \"[Run id: {}, {} trace iterations]\".format(\n            -1 if self.run_id is None else self.run_id,\n            len(self.trace_iterations),\n        )\n\n    def __iter__(self) -&gt; Iterator[OpenMLTraceIteration]:\n        yield from self.trace_iterations.values()\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.__init__","title":"<code>__init__(run_id, trace_iterations)</code>","text":"<p>Object to hold the trace content of a run.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Id for which the trace content is to be stored.</p> required <code>trace_iterations</code> <code>List[List]</code> <p>The trace content obtained by running a flow on a task.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>def __init__(\n    self,\n    run_id: int | None,\n    trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n):\n    \"\"\"Object to hold the trace content of a run.\n\n    Parameters\n    ----------\n    run_id : int\n        Id for which the trace content is to be stored.\n    trace_iterations : List[List]\n        The trace content obtained by running a flow on a task.\n    \"\"\"\n    self.run_id = run_id\n    self.trace_iterations = trace_iterations\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.generate","title":"<code>generate(attributes, content)</code>  <code>classmethod</code>","text":"<p>Generates an OpenMLRunTrace.</p> <p>Generates the trace object from the attributes and content extracted while running the underlying flow.</p> <p>Parameters:</p> Name Type Description Default <code>attributes</code> <code>list</code> <p>List of tuples describing the arff attributes.</p> required <code>content</code> <code>list</code> <p>List of lists containing information about the individual tuning runs.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef generate(\n    cls,\n    attributes: list[tuple[str, str]],\n    content: list[list[int | float | str]],\n) -&gt; OpenMLRunTrace:\n    \"\"\"Generates an OpenMLRunTrace.\n\n    Generates the trace object from the attributes and content extracted\n    while running the underlying flow.\n\n    Parameters\n    ----------\n    attributes : list\n        List of tuples describing the arff attributes.\n\n    content : list\n        List of lists containing information about the individual tuning\n        runs.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    if content is None:\n        raise ValueError(\"Trace content not available.\")\n    if attributes is None:\n        raise ValueError(\"Trace attributes not available.\")\n    if len(content) == 0:\n        raise ValueError(\"Trace content is empty.\")\n    if len(attributes) != len(content[0]):\n        raise ValueError(\n            \"Trace_attributes and trace_content not compatible:\"\n            f\" {attributes} vs {content[0]}\",\n        )\n\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not allowed when constructing a \"\n        \"trace object from run results.\",\n    )\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.get_selected_iteration","title":"<code>get_selected_iteration(fold, repeat)</code>","text":"<p>Returns the trace iteration that was marked as selected. In case multiple are marked as selected (should not happen) the first of these is returned</p> <p>Parameters:</p> Name Type Description Default <code>fold</code> <code>int</code> required <code>repeat</code> <code>int</code> required <p>Returns:</p> Type Description <code>int</code> <p>The trace iteration from the given fold and repeat that was selected as the best iteration by the search procedure</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n    \"\"\"\n    Returns the trace iteration that was marked as selected. In\n    case multiple are marked as selected (should not happen) the\n    first of these is returned\n\n    Parameters\n    ----------\n    fold: int\n\n    repeat: int\n\n    Returns\n    -------\n    int\n        The trace iteration from the given fold and repeat that was\n        selected as the best iteration by the search procedure\n    \"\"\"\n    for r, f, i in self.trace_iterations:\n        if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n            return i\n    raise ValueError(\n        \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n    )\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.merge_traces","title":"<code>merge_traces(traces)</code>  <code>classmethod</code>","text":"<p>Merge multiple traces into a single trace.</p> <p>Parameters:</p> Name Type Description Default <code>cls</code> <code>type</code> <p>Type of the trace object to be created.</p> required <code>traces</code> <code>List[OpenMLRunTrace]</code> <p>List of traces to merge.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> <p>A trace object representing the merged traces.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the parameters in the iterations of the traces being merged are not equal. If a key (repeat, fold, iteration) is encountered twice while merging the traces.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n    \"\"\"Merge multiple traces into a single trace.\n\n    Parameters\n    ----------\n    cls : type\n        Type of the trace object to be created.\n    traces : List[OpenMLRunTrace]\n        List of traces to merge.\n\n    Returns\n    -------\n    OpenMLRunTrace\n        A trace object representing the merged traces.\n\n    Raises\n    ------\n    ValueError\n        If the parameters in the iterations of the traces being merged are not equal.\n        If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n    \"\"\"\n    merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n    previous_iteration = None\n    for trace in traces:\n        for iteration in trace:\n            key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n            assert iteration.parameters is not None\n            param_keys = iteration.parameters.keys()\n\n            if previous_iteration is not None:\n                trace_itr = merged_trace[previous_iteration]\n\n                assert trace_itr.parameters is not None\n                trace_itr_keys = trace_itr.parameters.keys()\n\n                if list(param_keys) != list(trace_itr_keys):\n                    raise ValueError(\n                        \"Cannot merge traces because the parameters are not equal: \"\n                        \"{} vs {}\".format(\n                            list(trace_itr.parameters.keys()),\n                            list(iteration.parameters.keys()),\n                        ),\n                    )\n\n            if key in merged_trace:\n                raise ValueError(\n                    f\"Cannot merge traces because key '{key}' was encountered twice\",\n                )\n\n            merged_trace[key] = iteration\n            previous_iteration = key\n\n    return cls(None, merged_trace)\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.trace_from_arff","title":"<code>trace_from_arff(arff_obj)</code>  <code>classmethod</code>","text":"<p>Generate trace from arff trace.</p> <p>Creates a trace file from arff object (for example, generated by a local run).</p> <p>Parameters:</p> Name Type Description Default <code>arff_obj</code> <code>dict</code> <p>LIAC arff obj, dict containing attributes, relation, data.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from arff trace.\n\n    Creates a trace file from arff object (for example, generated by a\n    local run).\n\n    Parameters\n    ----------\n    arff_obj : dict\n        LIAC arff obj, dict containing attributes, relation, data.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    attributes = arff_obj[\"attributes\"]\n    content = arff_obj[\"data\"]\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not supported for arff serialization\",\n    )\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.trace_from_xml","title":"<code>trace_from_xml(xml)</code>  <code>classmethod</code>","text":"<p>Generate trace from xml.</p> <p>Creates a trace file from the xml description.</p> <p>Parameters:</p> Name Type Description Default <code>xml</code> <code>string | file-like object</code> <p>An xml description that can be either a <code>string</code> or a file-like object.</p> required <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRunTrace</code> <p>Object containing the run id and a dict containing the trace iterations.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from xml.\n\n    Creates a trace file from the xml description.\n\n    Parameters\n    ----------\n    xml : string | file-like object\n        An xml description that can be either a `string` or a file-like\n        object.\n\n    Returns\n    -------\n    run : OpenMLRunTrace\n        Object containing the run id and a dict containing the trace\n        iterations.\n    \"\"\"\n    if isinstance(xml, Path):\n        xml = str(xml.absolute())\n\n    result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n    run_id = result_dict[\"oml:run_id\"]\n    trace = OrderedDict()\n\n    if \"oml:trace_iteration\" not in result_dict:\n        raise ValueError(\"Run does not contain valid trace. \")\n    if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n        raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n    for itt in result_dict[\"oml:trace_iteration\"]:\n        repeat = int(itt[\"oml:repeat\"])\n        fold = int(itt[\"oml:fold\"])\n        iteration = int(itt[\"oml:iteration\"])\n        setup_string = json.loads(itt[\"oml:setup_string\"])\n        evaluation = float(itt[\"oml:evaluation\"])\n        selected_value = itt[\"oml:selected\"]\n        if selected_value == \"true\":\n            selected = True\n        elif selected_value == \"false\":\n            selected = False\n        else:\n            raise ValueError(\n                'expected {\"true\", \"false\"} value for '\n                \"selected field, received: %s\" % selected_value,\n            )\n\n        current = OpenMLTraceIteration(\n            repeat=repeat,\n            fold=fold,\n            iteration=iteration,\n            setup_string=setup_string,\n            evaluation=evaluation,\n            selected=selected,\n        )\n        trace[(repeat, fold, iteration)] = current\n\n    return cls(run_id, trace)\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.trace_to_arff","title":"<code>trace_to_arff()</code>","text":"<p>Generate the arff dictionary for uploading predictions to the server.</p> <p>Uses the trace object to generate an arff dictionary representation.</p> <p>Returns:</p> Name Type Description <code>arff_dict</code> <code>dict</code> <p>Dictionary representation of the ARFF file that will be uploaded. Contains information about the optimization trace.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def trace_to_arff(self) -&gt; dict[str, Any]:\n    \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n    Uses the trace object to generate an arff dictionary representation.\n\n    Returns\n    -------\n    arff_dict : dict\n        Dictionary representation of the ARFF file that will be uploaded.\n        Contains information about the optimization trace.\n    \"\"\"\n    if self.trace_iterations is None:\n        raise ValueError(\"trace_iterations missing from the trace object\")\n\n    # attributes that will be in trace arff\n    trace_attributes = [\n        (\"repeat\", \"NUMERIC\"),\n        (\"fold\", \"NUMERIC\"),\n        (\"iteration\", \"NUMERIC\"),\n        (\"evaluation\", \"NUMERIC\"),\n        (\"selected\", [\"true\", \"false\"]),\n    ]\n    trace_attributes.extend(\n        [\n            (PREFIX + parameter, \"STRING\")\n            for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n        ],\n    )\n\n    arff_dict: dict[str, Any] = {}\n    data = []\n    for trace_iteration in self.trace_iterations.values():\n        tmp_list = []\n        for _attr, _ in trace_attributes:\n            if _attr.startswith(PREFIX):\n                attr = _attr[len(PREFIX) :]\n                value = trace_iteration.get_parameters()[attr]\n            else:\n                attr = _attr\n                value = getattr(trace_iteration, attr)\n\n            if attr == \"selected\":\n                tmp_list.append(\"true\" if value else \"false\")\n            else:\n                tmp_list.append(value)\n        data.append(tmp_list)\n\n    arff_dict[\"attributes\"] = trace_attributes\n    arff_dict[\"data\"] = data\n    # TODO allow to pass a trace description when running a flow\n    arff_dict[\"relation\"] = \"Trace\"\n    return arff_dict\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLTraceIteration","title":"<code>OpenMLTraceIteration</code>  <code>dataclass</code>","text":"<p>OpenML Trace Iteration: parsed output from Run Trace call Exactly one of <code>setup_string</code> or <code>parameters</code> must be provided.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>repeat number (in case of no repeats: 0)</p> required <code>fold</code> <code>int</code> <p>fold number (in case of no folds: 0)</p> required <code>iteration</code> <code>int</code> <p>iteration number of optimization procedure</p> required <code>setup_string</code> <code>str</code> <p>json string representing the parameters If not provided, <code>parameters</code> should be set.</p> <code>None</code> <code>evaluation</code> <code>double</code> <p>The evaluation that was awarded to this trace iteration. Measure is defined by the task</p> required <code>selected</code> <code>bool</code> <p>Whether this was the best of all iterations, and hence selected for making predictions. Per fold/repeat there should be only one iteration selected</p> required <code>parameters</code> <code>OrderedDict</code> <p>Dictionary specifying parameter names and their values. If not provided, <code>setup_string</code> should be set.</p> <code>None</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@dataclass\nclass OpenMLTraceIteration:\n    \"\"\"\n    OpenML Trace Iteration: parsed output from Run Trace call\n    Exactly one of `setup_string` or `parameters` must be provided.\n\n    Parameters\n    ----------\n    repeat : int\n        repeat number (in case of no repeats: 0)\n\n    fold : int\n        fold number (in case of no folds: 0)\n\n    iteration : int\n        iteration number of optimization procedure\n\n    setup_string : str, optional\n        json string representing the parameters\n        If not provided, ``parameters`` should be set.\n\n    evaluation : double\n        The evaluation that was awarded to this trace iteration.\n        Measure is defined by the task\n\n    selected : bool\n        Whether this was the best of all iterations, and hence\n        selected for making predictions. Per fold/repeat there\n        should be only one iteration selected\n\n    parameters : OrderedDict, optional\n        Dictionary specifying parameter names and their values.\n        If not provided, ``setup_string`` should be set.\n    \"\"\"\n\n    repeat: int\n    fold: int\n    iteration: int\n\n    evaluation: float\n    selected: bool\n\n    setup_string: dict[str, str] | None = None\n    parameters: dict[str, str | int | float] | None = None\n\n    def __post_init__(self) -&gt; None:\n        # TODO: refactor into one argument of type &lt;str | OrderedDict&gt;\n        if self.setup_string and self.parameters:\n            raise ValueError(\n                \"Can only be instantiated with either `setup_string` or `parameters` argument.\",\n            )\n\n        if not (self.setup_string or self.parameters):\n            raise ValueError(\n                \"Either `setup_string` or `parameters` needs to be passed as argument.\",\n            )\n\n        if self.parameters is not None and not isinstance(self.parameters, dict):\n            raise TypeError(\n                \"argument parameters is not an instance of OrderedDict, but %s\"\n                % str(type(self.parameters)),\n            )\n\n    def get_parameters(self) -&gt; dict[str, Any]:\n        \"\"\"Get the parameters of this trace iteration.\"\"\"\n        # parameters have prefix 'parameter_'\n        if self.setup_string:\n            return {\n                param[len(PREFIX) :]: json.loads(value)\n                for param, value in self.setup_string.items()\n            }\n\n        assert self.parameters is not None\n        return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLTraceIteration.get_parameters","title":"<code>get_parameters()</code>","text":"<p>Get the parameters of this trace iteration.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_parameters(self) -&gt; dict[str, Any]:\n    \"\"\"Get the parameters of this trace iteration.\"\"\"\n    # parameters have prefix 'parameter_'\n    if self.setup_string:\n        return {\n            param[len(PREFIX) :]: json.loads(value)\n            for param, value in self.setup_string.items()\n        }\n\n    assert self.parameters is not None\n    return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/setups/","title":"setups","text":""},{"location":"reference/setups/#openml.setups.OpenMLParameter","title":"<code>OpenMLParameter</code>","text":"<p>Parameter object (used in setup).</p> <p>Parameters:</p> Name Type Description Default <code>input_id</code> <code>int</code> <p>The input id from the openml database</p> required <code>flow</code> <p>The flow to which this parameter is associated</p> required <code>flow</code> <p>The name of the flow (no version number) to which this parameter is associated</p> required <code>full_name</code> <code>str</code> <p>The name of the flow and parameter combined</p> required <code>parameter_name</code> <code>str</code> <p>The name of the parameter</p> required <code>data_type</code> <code>str</code> <p>The datatype of the parameter. generally unused for sklearn flows</p> required <code>default_value</code> <code>str</code> <p>The default value. For sklearn parameters, this is unknown and a default value is selected arbitrarily</p> required <code>value</code> <code>str</code> <p>If the parameter was set, the value that it was set to.</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLParameter:\n    \"\"\"Parameter object (used in setup).\n\n    Parameters\n    ----------\n    input_id : int\n        The input id from the openml database\n    flow id : int\n        The flow to which this parameter is associated\n    flow name : str\n        The name of the flow (no version number) to which this parameter\n        is associated\n    full_name : str\n        The name of the flow and parameter combined\n    parameter_name : str\n        The name of the parameter\n    data_type : str\n        The datatype of the parameter. generally unused for sklearn flows\n    default_value : str\n        The default value. For sklearn parameters, this is unknown and a\n        default value is selected arbitrarily\n    value : str\n        If the parameter was set, the value that it was set to.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        input_id: int,\n        flow_id: int,\n        flow_name: str,\n        full_name: str,\n        parameter_name: str,\n        data_type: str,\n        default_value: str,\n        value: str,\n    ):\n        self.id = input_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.full_name = full_name\n        self.parameter_name = parameter_name\n        self.data_type = data_type\n        self.default_value = default_value\n        self.value = value\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Parameter\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"ID\": self.id,\n            \"Flow ID\": self.flow_id,\n            # \"Flow Name\": self.flow_name,\n            \"Flow Name\": self.full_name,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Parameter Name\": self.parameter_name,\n        }\n        # indented prints for parameter attributes\n        # indention = 2 spaces + 1 | + 2 underscores\n        indent = \"{}|{}\".format(\" \" * 2, \"_\" * 2)\n        parameter_data_type = f\"{indent}Data Type\"\n        fields[parameter_data_type] = self.data_type\n        parameter_default = f\"{indent}Default\"\n        fields[parameter_default] = self.default_value\n        parameter_value = f\"{indent}Value\"\n        fields[parameter_value] = self.value\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Parameter Name\",\n            parameter_data_type,\n            parameter_default,\n            parameter_value,\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/setups/#openml.setups.OpenMLSetup","title":"<code>OpenMLSetup</code>","text":"<p>Setup object (a.k.a. Configuration).</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The OpenML setup id</p> required <code>flow_id</code> <code>int</code> <p>The flow that it is build upon</p> required <code>parameters</code> <code>dict</code> <p>The setting of the parameters</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLSetup:\n    \"\"\"Setup object (a.k.a. Configuration).\n\n    Parameters\n    ----------\n    setup_id : int\n        The OpenML setup id\n    flow_id : int\n        The flow that it is build upon\n    parameters : dict\n        The setting of the parameters\n    \"\"\"\n\n    def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None):\n        if not isinstance(setup_id, int):\n            raise ValueError(\"setup id should be int\")\n\n        if not isinstance(flow_id, int):\n            raise ValueError(\"flow id should be int\")\n\n        if parameters is not None and not isinstance(parameters, dict):\n            raise ValueError(\"parameters should be dict\")\n\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.parameters = parameters\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Setup\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Setup ID\": self.setup_id,\n            \"Flow ID\": self.flow_id,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"# of Parameters\": (\n                len(self.parameters) if self.parameters is not None else float(\"nan\")\n            ),\n        }\n\n        # determines the order in which the information will be printed\n        order = [\"Setup ID\", \"Flow ID\", \"Flow URL\", \"# of Parameters\"]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/setups/#openml.setups.get_setup","title":"<code>get_setup(setup_id)</code>","text":"<p>Downloads the setup (configuration) description from OpenML  and returns a structured object</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>OpenMLSetup (an initialized openml setup object)</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def get_setup(setup_id: int) -&gt; OpenMLSetup:\n    \"\"\"\n     Downloads the setup (configuration) description from OpenML\n     and returns a structured object\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    OpenMLSetup (an initialized openml setup object)\n    \"\"\"\n    setup_dir = Path(config.get_cache_directory()) / \"setups\" / str(setup_id)\n    setup_dir.mkdir(exist_ok=True, parents=True)\n\n    setup_file = setup_dir / \"description.xml\"\n\n    try:\n        return _get_cached_setup(setup_id)\n    except openml.exceptions.OpenMLCacheException:\n        url_suffix = \"/setup/%d\" % setup_id\n        setup_xml = openml._api_calls._perform_api_call(url_suffix, \"get\")\n        with setup_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(setup_xml)\n\n    result_dict = xmltodict.parse(setup_xml)\n    return _create_setup_from_xml(result_dict, output_format=\"object\")  # type: ignore\n</code></pre>"},{"location":"reference/setups/#openml.setups.initialize_model","title":"<code>initialize_model(setup_id)</code>","text":"<p>Initialized a model based on a setup_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def initialize_model(setup_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a setup_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    setup = get_setup(setup_id)\n    flow = openml.flows.get_flow(setup.flow_id)\n\n    # instead of using scikit-learns or any other library's \"set_params\" function, we override the\n    # OpenMLFlow objects default parameter value so we can utilize the\n    # Extension.flow_to_model() function to reinitialize the flow with the set defaults.\n    if setup.parameters is not None:\n        for hyperparameter in setup.parameters.values():\n            structure = flow.get_structure(\"flow_id\")\n            if len(structure[hyperparameter.flow_id]) &gt; 0:\n                subflow = flow.get_subflow(structure[hyperparameter.flow_id])\n            else:\n                subflow = flow\n            subflow.parameters[hyperparameter.parameter_name] = hyperparameter.value\n\n    return flow.extension.flow_to_model(flow)\n</code></pre>"},{"location":"reference/setups/#openml.setups.list_setups","title":"<code>list_setups(offset=None, size=None, flow=None, tag=None, setup=None, output_format='object')</code>","text":"<p>List all setups matching all of the given filters.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <code>None</code> <code>size</code> <code>int</code> <code>None</code> <code>flow</code> <code>int</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>setup</code> <code>Iterable[int]</code> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def list_setups(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    flow: int | None = None,\n    tag: str | None = None,\n    setup: Iterable[int] | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all setups matching all of the given filters.\n\n    Parameters\n    ----------\n    offset : int, optional\n    size : int, optional\n    flow : int, optional\n    tag : str, optional\n    setup : Iterable[int], optional\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict', 'object', or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    batch_size = 1000  # batch size for setups is lower\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_setups,\n        offset=offset,\n        size=size,\n        flow=flow,\n        tag=tag,\n        setup=setup,\n        batch_size=batch_size,\n    )\n</code></pre>"},{"location":"reference/setups/#openml.setups.setup_exists","title":"<code>setup_exists(flow)</code>","text":"<p>Checks whether a hyperparameter configuration already exists on the server.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>The openml flow object. Should have flow id present for the main flow and all subflows (i.e., it should be downloaded from the server by means of flow.get, and not instantiated locally)</p> required <p>Returns:</p> Name Type Description <code>setup_id</code> <code>int</code> <p>setup id iff exists, False otherwise</p> Source code in <code>openml/setups/functions.py</code> <pre><code>def setup_exists(flow: OpenMLFlow) -&gt; int:\n    \"\"\"\n    Checks whether a hyperparameter configuration already exists on the server.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        The openml flow object. Should have flow id present for the main flow\n        and all subflows (i.e., it should be downloaded from the server by\n        means of flow.get, and not instantiated locally)\n\n    Returns\n    -------\n    setup_id : int\n        setup id iff exists, False otherwise\n    \"\"\"\n    # sadly, this api call relies on a run object\n    openml.flows.functions._check_flow_for_server_id(flow)\n    if flow.model is None:\n        raise ValueError(\"Flow should have model field set with the actual model.\")\n    if flow.extension is None:\n        raise ValueError(\"Flow should have model field set with the correct extension.\")\n\n    # checks whether the flow exists on the server and flow ids align\n    exists = flow_exists(flow.name, flow.external_version)\n    if exists != flow.flow_id:\n        raise ValueError(\n            f\"Local flow id ({flow.id}) differs from server id ({exists}). \"\n            \"If this issue persists, please contact the developers.\",\n        )\n\n    openml_param_settings = flow.extension.obtain_parameter_values(flow)\n    description = xmltodict.unparse(_to_dict(flow.flow_id, openml_param_settings), pretty=True)\n    file_elements = {\n        \"description\": (\"description.arff\", description),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result = openml._api_calls._perform_api_call(\n        \"/setup/exists/\",\n        \"post\",\n        file_elements=file_elements,\n    )\n    result_dict = xmltodict.parse(result)\n    setup_id = int(result_dict[\"oml:setup_exists\"][\"oml:id\"])\n    return setup_id if setup_id &gt; 0 else False\n</code></pre>"},{"location":"reference/setups/functions/","title":"functions","text":""},{"location":"reference/setups/functions/#openml.setups.functions.__list_setups","title":"<code>__list_setups(api_call, output_format='object')</code>","text":"<p>Helper function to parse API calls which are lists of setups</p> Source code in <code>openml/setups/functions.py</code> <pre><code>def __list_setups(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\", \"object\"] = \"object\"\n) -&gt; dict[int, dict] | pd.DataFrame | dict[int, OpenMLSetup]:\n    \"\"\"Helper function to parse API calls which are lists of setups\"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    setups_dict = xmltodict.parse(xml_string, force_list=(\"oml:setup\",))\n    openml_uri = \"http://openml.org/openml\"\n    # Minimalistic check if the XML is useful\n    if \"oml:setups\" not in setups_dict:\n        raise ValueError(\n            'Error in return XML, does not contain \"oml:setups\":' \" %s\" % str(setups_dict),\n        )\n\n    if \"@xmlns:oml\" not in setups_dict[\"oml:setups\"]:\n        raise ValueError(\n            \"Error in return XML, does not contain \"\n            '\"oml:setups\"/@xmlns:oml: %s' % str(setups_dict),\n        )\n\n    if setups_dict[\"oml:setups\"][\"@xmlns:oml\"] != openml_uri:\n        raise ValueError(\n            \"Error in return XML, value of  \"\n            '\"oml:seyups\"/@xmlns:oml is not '\n            f'\"{openml_uri}\": {setups_dict!s}',\n        )\n\n    assert isinstance(setups_dict[\"oml:setups\"][\"oml:setup\"], list), type(setups_dict[\"oml:setups\"])\n\n    setups = {}\n    for setup_ in setups_dict[\"oml:setups\"][\"oml:setup\"]:\n        # making it a dict to give it the right format\n        current = _create_setup_from_xml(\n            {\"oml:setup_parameters\": setup_},\n            output_format=output_format,\n        )\n        if output_format == \"object\":\n            setups[current.setup_id] = current  # type: ignore\n        else:\n            setups[current[\"setup_id\"]] = current  # type: ignore\n\n    if output_format == \"dataframe\":\n        setups = pd.DataFrame.from_dict(setups, orient=\"index\")\n\n    return setups\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.get_setup","title":"<code>get_setup(setup_id)</code>","text":"<p>Downloads the setup (configuration) description from OpenML  and returns a structured object</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>OpenMLSetup (an initialized openml setup object)</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def get_setup(setup_id: int) -&gt; OpenMLSetup:\n    \"\"\"\n     Downloads the setup (configuration) description from OpenML\n     and returns a structured object\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    OpenMLSetup (an initialized openml setup object)\n    \"\"\"\n    setup_dir = Path(config.get_cache_directory()) / \"setups\" / str(setup_id)\n    setup_dir.mkdir(exist_ok=True, parents=True)\n\n    setup_file = setup_dir / \"description.xml\"\n\n    try:\n        return _get_cached_setup(setup_id)\n    except openml.exceptions.OpenMLCacheException:\n        url_suffix = \"/setup/%d\" % setup_id\n        setup_xml = openml._api_calls._perform_api_call(url_suffix, \"get\")\n        with setup_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(setup_xml)\n\n    result_dict = xmltodict.parse(setup_xml)\n    return _create_setup_from_xml(result_dict, output_format=\"object\")  # type: ignore\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.initialize_model","title":"<code>initialize_model(setup_id)</code>","text":"<p>Initialized a model based on a setup_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def initialize_model(setup_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a setup_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    setup = get_setup(setup_id)\n    flow = openml.flows.get_flow(setup.flow_id)\n\n    # instead of using scikit-learns or any other library's \"set_params\" function, we override the\n    # OpenMLFlow objects default parameter value so we can utilize the\n    # Extension.flow_to_model() function to reinitialize the flow with the set defaults.\n    if setup.parameters is not None:\n        for hyperparameter in setup.parameters.values():\n            structure = flow.get_structure(\"flow_id\")\n            if len(structure[hyperparameter.flow_id]) &gt; 0:\n                subflow = flow.get_subflow(structure[hyperparameter.flow_id])\n            else:\n                subflow = flow\n            subflow.parameters[hyperparameter.parameter_name] = hyperparameter.value\n\n    return flow.extension.flow_to_model(flow)\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.list_setups","title":"<code>list_setups(offset=None, size=None, flow=None, tag=None, setup=None, output_format='object')</code>","text":"<p>List all setups matching all of the given filters.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <code>None</code> <code>size</code> <code>int</code> <code>None</code> <code>flow</code> <code>int</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>setup</code> <code>Iterable[int]</code> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def list_setups(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    flow: int | None = None,\n    tag: str | None = None,\n    setup: Iterable[int] | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all setups matching all of the given filters.\n\n    Parameters\n    ----------\n    offset : int, optional\n    size : int, optional\n    flow : int, optional\n    tag : str, optional\n    setup : Iterable[int], optional\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict', 'object', or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    batch_size = 1000  # batch size for setups is lower\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_setups,\n        offset=offset,\n        size=size,\n        flow=flow,\n        tag=tag,\n        setup=setup,\n        batch_size=batch_size,\n    )\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.setup_exists","title":"<code>setup_exists(flow)</code>","text":"<p>Checks whether a hyperparameter configuration already exists on the server.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>The openml flow object. Should have flow id present for the main flow and all subflows (i.e., it should be downloaded from the server by means of flow.get, and not instantiated locally)</p> required <p>Returns:</p> Name Type Description <code>setup_id</code> <code>int</code> <p>setup id iff exists, False otherwise</p> Source code in <code>openml/setups/functions.py</code> <pre><code>def setup_exists(flow: OpenMLFlow) -&gt; int:\n    \"\"\"\n    Checks whether a hyperparameter configuration already exists on the server.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        The openml flow object. Should have flow id present for the main flow\n        and all subflows (i.e., it should be downloaded from the server by\n        means of flow.get, and not instantiated locally)\n\n    Returns\n    -------\n    setup_id : int\n        setup id iff exists, False otherwise\n    \"\"\"\n    # sadly, this api call relies on a run object\n    openml.flows.functions._check_flow_for_server_id(flow)\n    if flow.model is None:\n        raise ValueError(\"Flow should have model field set with the actual model.\")\n    if flow.extension is None:\n        raise ValueError(\"Flow should have model field set with the correct extension.\")\n\n    # checks whether the flow exists on the server and flow ids align\n    exists = flow_exists(flow.name, flow.external_version)\n    if exists != flow.flow_id:\n        raise ValueError(\n            f\"Local flow id ({flow.id}) differs from server id ({exists}). \"\n            \"If this issue persists, please contact the developers.\",\n        )\n\n    openml_param_settings = flow.extension.obtain_parameter_values(flow)\n    description = xmltodict.unparse(_to_dict(flow.flow_id, openml_param_settings), pretty=True)\n    file_elements = {\n        \"description\": (\"description.arff\", description),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result = openml._api_calls._perform_api_call(\n        \"/setup/exists/\",\n        \"post\",\n        file_elements=file_elements,\n    )\n    result_dict = xmltodict.parse(result)\n    setup_id = int(result_dict[\"oml:setup_exists\"][\"oml:id\"])\n    return setup_id if setup_id &gt; 0 else False\n</code></pre>"},{"location":"reference/setups/setup/","title":"setup","text":""},{"location":"reference/setups/setup/#openml.setups.setup.OpenMLParameter","title":"<code>OpenMLParameter</code>","text":"<p>Parameter object (used in setup).</p> <p>Parameters:</p> Name Type Description Default <code>input_id</code> <code>int</code> <p>The input id from the openml database</p> required <code>flow</code> <p>The flow to which this parameter is associated</p> required <code>flow</code> <p>The name of the flow (no version number) to which this parameter is associated</p> required <code>full_name</code> <code>str</code> <p>The name of the flow and parameter combined</p> required <code>parameter_name</code> <code>str</code> <p>The name of the parameter</p> required <code>data_type</code> <code>str</code> <p>The datatype of the parameter. generally unused for sklearn flows</p> required <code>default_value</code> <code>str</code> <p>The default value. For sklearn parameters, this is unknown and a default value is selected arbitrarily</p> required <code>value</code> <code>str</code> <p>If the parameter was set, the value that it was set to.</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLParameter:\n    \"\"\"Parameter object (used in setup).\n\n    Parameters\n    ----------\n    input_id : int\n        The input id from the openml database\n    flow id : int\n        The flow to which this parameter is associated\n    flow name : str\n        The name of the flow (no version number) to which this parameter\n        is associated\n    full_name : str\n        The name of the flow and parameter combined\n    parameter_name : str\n        The name of the parameter\n    data_type : str\n        The datatype of the parameter. generally unused for sklearn flows\n    default_value : str\n        The default value. For sklearn parameters, this is unknown and a\n        default value is selected arbitrarily\n    value : str\n        If the parameter was set, the value that it was set to.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        input_id: int,\n        flow_id: int,\n        flow_name: str,\n        full_name: str,\n        parameter_name: str,\n        data_type: str,\n        default_value: str,\n        value: str,\n    ):\n        self.id = input_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.full_name = full_name\n        self.parameter_name = parameter_name\n        self.data_type = data_type\n        self.default_value = default_value\n        self.value = value\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Parameter\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"ID\": self.id,\n            \"Flow ID\": self.flow_id,\n            # \"Flow Name\": self.flow_name,\n            \"Flow Name\": self.full_name,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Parameter Name\": self.parameter_name,\n        }\n        # indented prints for parameter attributes\n        # indention = 2 spaces + 1 | + 2 underscores\n        indent = \"{}|{}\".format(\" \" * 2, \"_\" * 2)\n        parameter_data_type = f\"{indent}Data Type\"\n        fields[parameter_data_type] = self.data_type\n        parameter_default = f\"{indent}Default\"\n        fields[parameter_default] = self.default_value\n        parameter_value = f\"{indent}Value\"\n        fields[parameter_value] = self.value\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Parameter Name\",\n            parameter_data_type,\n            parameter_default,\n            parameter_value,\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/setups/setup/#openml.setups.setup.OpenMLSetup","title":"<code>OpenMLSetup</code>","text":"<p>Setup object (a.k.a. Configuration).</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The OpenML setup id</p> required <code>flow_id</code> <code>int</code> <p>The flow that it is build upon</p> required <code>parameters</code> <code>dict</code> <p>The setting of the parameters</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLSetup:\n    \"\"\"Setup object (a.k.a. Configuration).\n\n    Parameters\n    ----------\n    setup_id : int\n        The OpenML setup id\n    flow_id : int\n        The flow that it is build upon\n    parameters : dict\n        The setting of the parameters\n    \"\"\"\n\n    def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None):\n        if not isinstance(setup_id, int):\n            raise ValueError(\"setup id should be int\")\n\n        if not isinstance(flow_id, int):\n            raise ValueError(\"flow id should be int\")\n\n        if parameters is not None and not isinstance(parameters, dict):\n            raise ValueError(\"parameters should be dict\")\n\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.parameters = parameters\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Setup\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Setup ID\": self.setup_id,\n            \"Flow ID\": self.flow_id,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"# of Parameters\": (\n                len(self.parameters) if self.parameters is not None else float(\"nan\")\n            ),\n        }\n\n        # determines the order in which the information will be printed\n        order = [\"Setup ID\", \"Flow ID\", \"Flow URL\", \"# of Parameters\"]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/study/","title":"study","text":""},{"location":"reference/study/#openml.study.OpenMLBenchmarkSuite","title":"<code>OpenMLBenchmarkSuite</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).</p> <p>It contains the following information: name, id, description, creation date, creator id and the task ids.</p> <p>According to this list of task ids, the suite object receives a list of OpenML object ids (datasets).</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLBenchmarkSuite(BaseStudy):\n    \"\"\"\n    An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and the task ids.\n\n    According to this list of task ids, the suite object receives a list of\n    OpenML object ids (datasets).\n\n    Parameters\n    ----------\n    suite_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        suite_id: int | None,\n        alias: str | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n    ):\n        super().__init__(\n            study_id=suite_id,\n            alias=alias,\n            main_entity_type=\"task\",\n            benchmark_suite=None,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=None,\n            runs=None,\n            setups=None,\n        )\n</code></pre>"},{"location":"reference/study/#openml.study.OpenMLStudy","title":"<code>OpenMLStudy</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLStudy represents the OpenML concept of a study (a collection of runs).</p> <p>It contains the following information: name, id, description, creation date, creator id and a list of run ids.</p> <p>According to this list of run ids, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLStudy(BaseStudy):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study (a collection of runs).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and a list of run ids.\n\n    According to this list of run ids, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        super().__init__(\n            study_id=study_id,\n            alias=alias,\n            main_entity_type=\"run\",\n            benchmark_suite=benchmark_suite,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=flows,\n            runs=runs,\n            setups=setups,\n        )\n</code></pre>"},{"location":"reference/study/#openml.study.attach_to_study","title":"<code>attach_to_study(study_id, run_ids)</code>","text":"<p>Attaches a set of runs to a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of runs to a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    result_xml = openml._api_calls._perform_api_call(\n        call=f\"study/{study_id}/attach\",\n        request_method=\"post\",\n        data={\"ids\": \",\".join(str(x) for x in run_ids)},\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_attach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/#openml.study.attach_to_suite","title":"<code>attach_to_suite(suite_id, task_ids)</code>","text":"<p>Attaches a set of tasks to a benchmarking suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the suite (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of tasks to a benchmarking suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the suite (in terms of explicitly linked entities)\n    \"\"\"\n    return attach_to_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/#openml.study.create_benchmark_suite","title":"<code>create_benchmark_suite(name, description, task_ids, alias=None)</code>","text":"<p>Creates an OpenML benchmark suite (collection of entity types, where the tasks are the linked entity)</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>task_ids</code> <code>list</code> <p>a list of task ids associated with this study more can be added later with <code>attach_to_suite</code>.</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_benchmark_suite(\n    name: str,\n    description: str,\n    task_ids: list[int],\n    alias: str | None = None,\n) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Creates an OpenML benchmark suite (collection of entity types, where\n    the tasks are the linked entity)\n\n    Parameters\n    ----------\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    task_ids : list\n        a list of task ids associated with this study\n        more can be added later with ``attach_to_suite``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLBenchmarkSuite(\n        suite_id=None,\n        alias=alias,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=task_ids,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.create_study","title":"<code>create_study(name, description, run_ids=None, alias=None, benchmark_suite=None)</code>","text":"<p>Creates an OpenML study (collection of data, tasks, flows, setups and run), where the runs are the main entity (collection consists of runs and all entities (flows, tasks, etc) that are related to these runs)</p> <p>Parameters:</p> Name Type Description Default <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran.</p> <code>None</code> <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>run_ids</code> <code>list</code> <p>a list of run ids associated with this study, these can also be added later with <code>attach_to_study</code>.</p> <code>None</code> <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <code>benchmark_suite</code> <code>int | None</code> <p>the ID of the suite for which this study contains run results</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_study(\n    name: str,\n    description: str,\n    run_ids: list[int] | None = None,\n    alias: str | None = None,\n    benchmark_suite: int | None = None,\n) -&gt; OpenMLStudy:\n    \"\"\"\n    Creates an OpenML study (collection of data, tasks, flows, setups and run),\n    where the runs are the main entity (collection consists of runs and all\n    entities (flows, tasks, etc) that are related to these runs)\n\n    Parameters\n    ----------\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    run_ids : list, optional\n        a list of run ids associated with this study,\n        these can also be added later with ``attach_to_study``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite: int (optional)\n        the ID of the suite for which this study contains run results\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLStudy(\n        study_id=None,\n        alias=alias,\n        benchmark_suite=benchmark_suite,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=None,\n        flows=None,\n        runs=run_ids if run_ids != [] else None,\n        setups=None,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.delete_study","title":"<code>delete_study(study_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_study(study_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return openml.utils._delete_entity(\"study\", study_id)\n</code></pre>"},{"location":"reference/study/#openml.study.delete_suite","title":"<code>delete_suite(suite_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_suite(suite_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return delete_study(suite_id)\n</code></pre>"},{"location":"reference/study/#openml.study.detach_from_study","title":"<code>detach_from_study(study_id, run_ids)</code>","text":"<p>Detaches a set of run ids from a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of run ids from a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    uri = \"study/%d/detach\" % study_id\n    post_variables = {\"ids\": \",\".join(str(x) for x in run_ids)}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        call=uri,\n        request_method=\"post\",\n        data=post_variables,\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_detach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/#openml.study.detach_from_suite","title":"<code>detach_from_suite(suite_id, task_ids)</code>","text":"<p>Detaches a set of task ids from a suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <code>new size of the study (in terms of explicitly linked entities)</code> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of task ids from a suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n    new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    return detach_from_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/#openml.study.get_study","title":"<code>get_study(study_id, arg_for_backwards_compat=None)</code>","text":"<p>Retrieves all relevant information of an OpenML study from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <code>arg_for_backwards_compat</code> <code>str</code> <p>The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the API which required specifying the type of study, i.e. tasks. We changed the implementation of studies since then and split them up into suites (collections of tasks) and studies (collections of runs) so this argument is no longer needed.</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>The OpenML study object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_study(\n    study_id: int | str,\n    arg_for_backwards_compat: str | None = None,  # noqa: ARG001\n) -&gt; OpenMLStudy:  # F401\n    \"\"\"\n    Retrieves all relevant information of an OpenML study from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    arg_for_backwards_compat : str, optional\n        The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the\n        API which required specifying the type of study, i.e. tasks. We changed the\n        implementation of studies since then and split them up into suites (collections of tasks)\n        and studies (collections of runs) so this argument is no longer needed.\n\n    Returns\n    -------\n    OpenMLStudy\n        The OpenML study object\n    \"\"\"\n    if study_id == \"OpenML100\":\n        message = (\n            \"It looks like you are running code from the OpenML100 paper. It still works, but lots \"\n            \"of things have changed since then. Please use `get_suite('OpenML100')` instead.\"\n        )\n        warnings.warn(message, DeprecationWarning, stacklevel=2)\n        openml.config.logger.warning(message)\n        study = _get_study(study_id, entity_type=\"task\")\n        assert isinstance(study, OpenMLBenchmarkSuite)\n\n        return study  # type: ignore\n\n    study = _get_study(study_id, entity_type=\"run\")\n    assert isinstance(study, OpenMLStudy)\n    return study\n</code></pre>"},{"location":"reference/study/#openml.study.get_suite","title":"<code>get_suite(suite_id)</code>","text":"<p>Retrieves all relevant information of an OpenML benchmarking suite from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <p>Returns:</p> Type Description <code>OpenMLSuite</code> <p>The OpenML suite object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_suite(suite_id: int | str) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Retrieves all relevant information of an OpenML benchmarking suite from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    Returns\n    -------\n    OpenMLSuite\n        The OpenML suite object\n    \"\"\"\n    study = _get_study(suite_id, entity_type=\"task\")\n    assert isinstance(study, OpenMLBenchmarkSuite)\n\n    return study\n</code></pre>"},{"location":"reference/study/#openml.study.list_studies","title":"<code>list_studies(offset=None, size=None, status=None, uploader=None, benchmark_suite=None, output_format='dict')</code>","text":"<pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all studies which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of studies to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of studies to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active studies are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return studies created by these users.</p> <code>None</code> <code>benchmark_suite</code> <code>int</code> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> <li> <p>If output_format='dataframe'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_studies(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[str] | None = None,\n    benchmark_suite: int | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all studies which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of studies to skip, starting from the first.\n    size : int, optional\n        The maximum number of studies to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        studies are returned.\n    uploader : list (int), optional\n        Result filter. Will only return studies created by these users.\n    benchmark_suite : int, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"run\",\n        status=status,\n        uploader=uploader,\n        benchmark_suite=benchmark_suite,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.list_suites","title":"<code>list_suites(offset=None, size=None, status=None, uploader=None, output_format='dict')</code>","text":"<pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all suites which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of suites to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of suites to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active suites are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return suites created by these users.</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every suite is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> <li> <p>If output_format='dataframe'     Every row is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_suites(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[int] | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all suites which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of suites to skip, starting from the first.\n    size : int, optional\n        The maximum number of suites to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        suites are returned.\n    uploader : list (int), optional\n        Result filter. Will only return suites created by these users.\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every suite is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n\n        - If output_format='dataframe'\n            Every row is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"task\",\n        status=status,\n        uploader=uploader,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.update_study_status","title":"<code>update_study_status(study_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_study_status(study_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    study_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(\"Illegal status value. \" \"Legal values: %s\" % legal_status)\n    data = {\"study_id\": study_id, \"status\": status}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"study/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_study_id = result[\"oml:study_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:study_status_update\"][\"oml:status\"]\n    if status != server_status or int(study_id) != int(server_study_id):\n        # This should never happen\n        raise ValueError(\"Study id/status does not collide\")\n</code></pre>"},{"location":"reference/study/#openml.study.update_suite_status","title":"<code>update_suite_status(suite_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_suite_status(suite_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    suite_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    return update_study_status(suite_id, status)\n</code></pre>"},{"location":"reference/study/functions/","title":"functions","text":""},{"location":"reference/study/functions/#openml.study.functions.__list_studies","title":"<code>__list_studies(api_call, output_format='dict')</code>","text":"<pre><code>__list_studies(api_call: str, output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>__list_studies(api_call: str, output_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Retrieves the list of OpenML studies and returns it in a dictionary or a Pandas DataFrame.</p> <p>Parameters:</p> Name Type Description Default <code>api_call</code> <code>str</code> <p>The API call for retrieving the list of OpenML studies.</p> required <code>output_format</code> <code>str in {'dict', 'dataframe'}</code> <p>Format of the output, either 'object' for a dictionary or 'dataframe' for a Pandas DataFrame.</p> <code>'dict'</code> <p>Returns:</p> Type Description <code>Union[Dict, DataFrame]</code> <p>A dictionary or Pandas DataFrame of OpenML studies, depending on the value of 'output_format'.</p> Source code in <code>openml/study/functions.py</code> <pre><code>def __list_studies(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\"] = \"dict\"\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Retrieves the list of OpenML studies and\n    returns it in a dictionary or a Pandas DataFrame.\n\n    Parameters\n    ----------\n    api_call : str\n        The API call for retrieving the list of OpenML studies.\n    output_format : str in {\"dict\", \"dataframe\"}\n        Format of the output, either 'object' for a dictionary\n        or 'dataframe' for a Pandas DataFrame.\n\n    Returns\n    -------\n    Union[Dict, pd.DataFrame]\n        A dictionary or Pandas DataFrame of OpenML studies,\n        depending on the value of 'output_format'.\n    \"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    study_dict = xmltodict.parse(xml_string, force_list=(\"oml:study\",))\n\n    # Minimalistic check if the XML is useful\n    assert isinstance(study_dict[\"oml:study_list\"][\"oml:study\"], list), type(\n        study_dict[\"oml:study_list\"],\n    )\n    assert study_dict[\"oml:study_list\"][\"@xmlns:oml\"] == \"http://openml.org/openml\", study_dict[\n        \"oml:study_list\"\n    ][\"@xmlns:oml\"]\n\n    studies = {}\n    for study_ in study_dict[\"oml:study_list\"][\"oml:study\"]:\n        # maps from xml name to a tuple of (dict name, casting fn)\n        expected_fields = {\n            \"oml:id\": (\"id\", int),\n            \"oml:alias\": (\"alias\", str),\n            \"oml:main_entity_type\": (\"main_entity_type\", str),\n            \"oml:benchmark_suite\": (\"benchmark_suite\", int),\n            \"oml:name\": (\"name\", str),\n            \"oml:status\": (\"status\", str),\n            \"oml:creation_date\": (\"creation_date\", str),\n            \"oml:creator\": (\"creator\", int),\n        }\n        study_id = int(study_[\"oml:id\"])\n        current_study = {}\n        for oml_field_name, (real_field_name, cast_fn) in expected_fields.items():\n            if oml_field_name in study_:\n                current_study[real_field_name] = cast_fn(study_[oml_field_name])\n        current_study[\"id\"] = int(current_study[\"id\"])\n        studies[study_id] = current_study\n\n    if output_format == \"dataframe\":\n        studies = pd.DataFrame.from_dict(studies, orient=\"index\")\n    return studies\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.attach_to_study","title":"<code>attach_to_study(study_id, run_ids)</code>","text":"<p>Attaches a set of runs to a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of runs to a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    result_xml = openml._api_calls._perform_api_call(\n        call=f\"study/{study_id}/attach\",\n        request_method=\"post\",\n        data={\"ids\": \",\".join(str(x) for x in run_ids)},\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_attach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.attach_to_suite","title":"<code>attach_to_suite(suite_id, task_ids)</code>","text":"<p>Attaches a set of tasks to a benchmarking suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the suite (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of tasks to a benchmarking suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the suite (in terms of explicitly linked entities)\n    \"\"\"\n    return attach_to_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.create_benchmark_suite","title":"<code>create_benchmark_suite(name, description, task_ids, alias=None)</code>","text":"<p>Creates an OpenML benchmark suite (collection of entity types, where the tasks are the linked entity)</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>task_ids</code> <code>list</code> <p>a list of task ids associated with this study more can be added later with <code>attach_to_suite</code>.</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_benchmark_suite(\n    name: str,\n    description: str,\n    task_ids: list[int],\n    alias: str | None = None,\n) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Creates an OpenML benchmark suite (collection of entity types, where\n    the tasks are the linked entity)\n\n    Parameters\n    ----------\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    task_ids : list\n        a list of task ids associated with this study\n        more can be added later with ``attach_to_suite``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLBenchmarkSuite(\n        suite_id=None,\n        alias=alias,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=task_ids,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.create_study","title":"<code>create_study(name, description, run_ids=None, alias=None, benchmark_suite=None)</code>","text":"<p>Creates an OpenML study (collection of data, tasks, flows, setups and run), where the runs are the main entity (collection consists of runs and all entities (flows, tasks, etc) that are related to these runs)</p> <p>Parameters:</p> Name Type Description Default <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran.</p> <code>None</code> <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>run_ids</code> <code>list</code> <p>a list of run ids associated with this study, these can also be added later with <code>attach_to_study</code>.</p> <code>None</code> <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <code>benchmark_suite</code> <code>int | None</code> <p>the ID of the suite for which this study contains run results</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_study(\n    name: str,\n    description: str,\n    run_ids: list[int] | None = None,\n    alias: str | None = None,\n    benchmark_suite: int | None = None,\n) -&gt; OpenMLStudy:\n    \"\"\"\n    Creates an OpenML study (collection of data, tasks, flows, setups and run),\n    where the runs are the main entity (collection consists of runs and all\n    entities (flows, tasks, etc) that are related to these runs)\n\n    Parameters\n    ----------\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    run_ids : list, optional\n        a list of run ids associated with this study,\n        these can also be added later with ``attach_to_study``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite: int (optional)\n        the ID of the suite for which this study contains run results\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLStudy(\n        study_id=None,\n        alias=alias,\n        benchmark_suite=benchmark_suite,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=None,\n        flows=None,\n        runs=run_ids if run_ids != [] else None,\n        setups=None,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.delete_study","title":"<code>delete_study(study_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_study(study_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return openml.utils._delete_entity(\"study\", study_id)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.delete_suite","title":"<code>delete_suite(suite_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_suite(suite_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return delete_study(suite_id)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.detach_from_study","title":"<code>detach_from_study(study_id, run_ids)</code>","text":"<p>Detaches a set of run ids from a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of run ids from a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    uri = \"study/%d/detach\" % study_id\n    post_variables = {\"ids\": \",\".join(str(x) for x in run_ids)}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        call=uri,\n        request_method=\"post\",\n        data=post_variables,\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_detach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.detach_from_suite","title":"<code>detach_from_suite(suite_id, task_ids)</code>","text":"<p>Detaches a set of task ids from a suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <code>new size of the study (in terms of explicitly linked entities)</code> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of task ids from a suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n    new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    return detach_from_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.get_study","title":"<code>get_study(study_id, arg_for_backwards_compat=None)</code>","text":"<p>Retrieves all relevant information of an OpenML study from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <code>arg_for_backwards_compat</code> <code>str</code> <p>The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the API which required specifying the type of study, i.e. tasks. We changed the implementation of studies since then and split them up into suites (collections of tasks) and studies (collections of runs) so this argument is no longer needed.</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>The OpenML study object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_study(\n    study_id: int | str,\n    arg_for_backwards_compat: str | None = None,  # noqa: ARG001\n) -&gt; OpenMLStudy:  # F401\n    \"\"\"\n    Retrieves all relevant information of an OpenML study from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    arg_for_backwards_compat : str, optional\n        The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the\n        API which required specifying the type of study, i.e. tasks. We changed the\n        implementation of studies since then and split them up into suites (collections of tasks)\n        and studies (collections of runs) so this argument is no longer needed.\n\n    Returns\n    -------\n    OpenMLStudy\n        The OpenML study object\n    \"\"\"\n    if study_id == \"OpenML100\":\n        message = (\n            \"It looks like you are running code from the OpenML100 paper. It still works, but lots \"\n            \"of things have changed since then. Please use `get_suite('OpenML100')` instead.\"\n        )\n        warnings.warn(message, DeprecationWarning, stacklevel=2)\n        openml.config.logger.warning(message)\n        study = _get_study(study_id, entity_type=\"task\")\n        assert isinstance(study, OpenMLBenchmarkSuite)\n\n        return study  # type: ignore\n\n    study = _get_study(study_id, entity_type=\"run\")\n    assert isinstance(study, OpenMLStudy)\n    return study\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.get_suite","title":"<code>get_suite(suite_id)</code>","text":"<p>Retrieves all relevant information of an OpenML benchmarking suite from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <p>Returns:</p> Type Description <code>OpenMLSuite</code> <p>The OpenML suite object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_suite(suite_id: int | str) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Retrieves all relevant information of an OpenML benchmarking suite from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    Returns\n    -------\n    OpenMLSuite\n        The OpenML suite object\n    \"\"\"\n    study = _get_study(suite_id, entity_type=\"task\")\n    assert isinstance(study, OpenMLBenchmarkSuite)\n\n    return study\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.list_studies","title":"<code>list_studies(offset=None, size=None, status=None, uploader=None, benchmark_suite=None, output_format='dict')</code>","text":"<pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all studies which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of studies to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of studies to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active studies are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return studies created by these users.</p> <code>None</code> <code>benchmark_suite</code> <code>int</code> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> <li> <p>If output_format='dataframe'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_studies(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[str] | None = None,\n    benchmark_suite: int | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all studies which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of studies to skip, starting from the first.\n    size : int, optional\n        The maximum number of studies to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        studies are returned.\n    uploader : list (int), optional\n        Result filter. Will only return studies created by these users.\n    benchmark_suite : int, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"run\",\n        status=status,\n        uploader=uploader,\n        benchmark_suite=benchmark_suite,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.list_suites","title":"<code>list_suites(offset=None, size=None, status=None, uploader=None, output_format='dict')</code>","text":"<pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all suites which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of suites to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of suites to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active suites are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return suites created by these users.</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every suite is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> <li> <p>If output_format='dataframe'     Every row is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_suites(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[int] | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all suites which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of suites to skip, starting from the first.\n    size : int, optional\n        The maximum number of suites to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        suites are returned.\n    uploader : list (int), optional\n        Result filter. Will only return suites created by these users.\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every suite is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n\n        - If output_format='dataframe'\n            Every row is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"task\",\n        status=status,\n        uploader=uploader,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.update_study_status","title":"<code>update_study_status(study_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_study_status(study_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    study_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(\"Illegal status value. \" \"Legal values: %s\" % legal_status)\n    data = {\"study_id\": study_id, \"status\": status}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"study/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_study_id = result[\"oml:study_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:study_status_update\"][\"oml:status\"]\n    if status != server_status or int(study_id) != int(server_study_id):\n        # This should never happen\n        raise ValueError(\"Study id/status does not collide\")\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.update_suite_status","title":"<code>update_suite_status(suite_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_suite_status(suite_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    suite_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    return update_study_status(suite_id, status)\n</code></pre>"},{"location":"reference/study/study/","title":"study","text":""},{"location":"reference/study/study/#openml.study.study.BaseStudy","title":"<code>BaseStudy</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>An OpenMLStudy represents the OpenML concept of a study. It contains the following information: name, id, description, creation date, creator id and a set of tags.</p> <p>According to this list of tags, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Can be used to obtain all relevant information from a study at once.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class BaseStudy(OpenMLBase):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study. It contains\n    the following information: name, id, description, creation date,\n    creator id and a set of tags.\n\n    According to this list of tags, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Can be used to obtain all relevant information from a study at once.\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        main_entity_type: str,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        self.study_id = study_id\n        self.alias = alias\n        self.main_entity_type = main_entity_type\n        self.benchmark_suite = benchmark_suite\n        self.name = name\n        self.description = description\n        self.status = status\n        self.creation_date = creation_date\n        self.creator = creator\n        self.tags = tags  # LEGACY. Can be removed soon\n        self.data = data\n        self.tasks = tasks\n        self.flows = flows\n        self.setups = setups\n        self.runs = runs\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"s\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the id of the study.\"\"\"\n        return self.study_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields: dict[str, Any] = {\n            \"Name\": self.name,\n            \"Status\": self.status,\n            \"Main Entity Type\": self.main_entity_type,\n        }\n        if self.study_id is not None:\n            fields[\"ID\"] = self.study_id\n            fields[\"Study URL\"] = self.openml_url\n        if self.creator is not None:\n            fields[\"Creator\"] = f\"{get_server_base_url()}/u/{self.creator}\"\n        if self.creation_date is not None:\n            fields[\"Upload Time\"] = self.creation_date.replace(\"T\", \" \")\n        if self.data is not None:\n            fields[\"# of Data\"] = len(self.data)\n        if self.tasks is not None:\n            fields[\"# of Tasks\"] = len(self.tasks)\n        if self.flows is not None:\n            fields[\"# of Flows\"] = len(self.flows)\n        if self.runs is not None:\n            fields[\"# of Runs\"] = len(self.runs)\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Name\",\n            \"Status\",\n            \"Main Entity Type\",\n            \"Study URL\",\n            \"# of Data\",\n            \"# of Tasks\",\n            \"# of Flows\",\n            \"# of Runs\",\n            \"Creator\",\n            \"Upload Time\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.study_id = int(xml_response[\"oml:study_upload\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        # some can not be uploaded, e.g., id, creator, creation_date\n        simple_props = [\"alias\", \"main_entity_type\", \"name\", \"description\"]\n\n        # TODO(eddiebergman): Begging for a walrus if we can drop 3.7\n        simple_prop_values = {}\n        for prop_name in simple_props:\n            content = getattr(self, prop_name, None)\n            if content is not None:\n                simple_prop_values[\"oml:\" + prop_name] = content\n\n        # maps from attribute name (which is used as outer tag name) to immer\n        # tag name e.g., self.tasks -&gt; &lt;oml:tasks&gt;&lt;oml:task_id&gt;1987&lt;/oml:task_id&gt;&lt;/oml:tasks&gt;\n        complex_props = {\"tasks\": \"task_id\", \"runs\": \"run_id\"}\n\n        # TODO(eddiebergman): Begging for a walrus if we can drop 3.7\n        complex_prop_values = {}\n        for prop_name, inner_name in complex_props.items():\n            content = getattr(self, prop_name, None)\n            if content is not None:\n                complex_prop_values[\"oml:\" + prop_name] = {\"oml:\" + inner_name: content}\n\n        return {\n            \"oml:study\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **simple_prop_values,\n                **complex_prop_values,\n            }\n        }\n\n    def push_tag(self, tag: str) -&gt; None:\n        \"\"\"Add a tag to the study.\"\"\"\n        raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n\n    def remove_tag(self, tag: str) -&gt; None:\n        \"\"\"Remove a tag from the study.\"\"\"\n        raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.BaseStudy.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the id of the study.</p>"},{"location":"reference/study/study/#openml.study.study.BaseStudy.push_tag","title":"<code>push_tag(tag)</code>","text":"<p>Add a tag to the study.</p> Source code in <code>openml/study/study.py</code> <pre><code>def push_tag(self, tag: str) -&gt; None:\n    \"\"\"Add a tag to the study.\"\"\"\n    raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.BaseStudy.remove_tag","title":"<code>remove_tag(tag)</code>","text":"<p>Remove a tag from the study.</p> Source code in <code>openml/study/study.py</code> <pre><code>def remove_tag(self, tag: str) -&gt; None:\n    \"\"\"Remove a tag from the study.\"\"\"\n    raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.OpenMLBenchmarkSuite","title":"<code>OpenMLBenchmarkSuite</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).</p> <p>It contains the following information: name, id, description, creation date, creator id and the task ids.</p> <p>According to this list of task ids, the suite object receives a list of OpenML object ids (datasets).</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLBenchmarkSuite(BaseStudy):\n    \"\"\"\n    An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and the task ids.\n\n    According to this list of task ids, the suite object receives a list of\n    OpenML object ids (datasets).\n\n    Parameters\n    ----------\n    suite_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        suite_id: int | None,\n        alias: str | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n    ):\n        super().__init__(\n            study_id=suite_id,\n            alias=alias,\n            main_entity_type=\"task\",\n            benchmark_suite=None,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=None,\n            runs=None,\n            setups=None,\n        )\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.OpenMLStudy","title":"<code>OpenMLStudy</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLStudy represents the OpenML concept of a study (a collection of runs).</p> <p>It contains the following information: name, id, description, creation date, creator id and a list of run ids.</p> <p>According to this list of run ids, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLStudy(BaseStudy):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study (a collection of runs).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and a list of run ids.\n\n    According to this list of run ids, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        super().__init__(\n            study_id=study_id,\n            alias=alias,\n            main_entity_type=\"run\",\n            benchmark_suite=benchmark_suite,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=flows,\n            runs=runs,\n            setups=setups,\n        )\n</code></pre>"},{"location":"reference/tasks/","title":"tasks","text":""},{"location":"reference/tasks/#openml.tasks.OpenMLClassificationTask","title":"<code>OpenMLClassificationTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Classification task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the Classification task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the Classification task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target variable.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the Classification task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the Classification task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the Classification task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Classification task (if it already exists on OpenML).</p> <code>None</code> <code>class_labels</code> <code>List of str</code> <p>A list of class labels (for classification tasks).</p> <code>None</code> <code>cost_matrix</code> <code>array</code> <p>A cost matrix (for classification tasks).</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClassificationTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Classification task type.\n    task_type : str\n        Name of the Classification task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the Classification task.\n    target_name : str\n        Name of the target variable.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the Classification task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the Classification task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure.\n    data_splits_url : str, default=None\n        URL of the data splits for the Classification task.\n    task_id : Union[int, None]\n        ID of the Classification task (if it already exists on OpenML).\n    class_labels : List of str, default=None\n        A list of class labels (for classification tasks).\n    cost_matrix : array, default=None\n        A cost matrix (for classification tasks).\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n        self.class_labels = class_labels\n        self.cost_matrix = cost_matrix\n\n        if cost_matrix is not None:\n            raise NotImplementedError(\"Costmatrix\")\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLClusteringTask","title":"<code>OpenMLClusteringTask</code>","text":"<p>               Bases: <code>OpenMLTask</code></p> <p>OpenML Clustering object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML clustering task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML clustering task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset used in clustering the task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML clustering task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure used in the clustering task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the clustering task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the clustering task.</p> <code>None</code> <code>target_name</code> <code>str</code> <p>Name of the target feature (class) that is not part of the feature set for the clustering task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClusteringTask(OpenMLTask):\n    \"\"\"OpenML Clustering object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML clustering task.\n    task_type : str\n        Task type of the OpenML clustering task.\n    data_set_id : int\n        ID of the OpenML dataset used in clustering the task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    task_id : Union[int, None]\n        ID of the OpenML clustering task.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure used in the clustering task.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the clustering task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the clustering task.\n    target_name : str, default=None\n        Name of the target feature (class) that is not part of the\n        feature set for the clustering task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 17,\n        task_id: int | None = None,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        evaluation_measure: str | None = None,\n        target_name: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            evaluation_measure=evaluation_measure,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\"] = \"array\",\n    ) -&gt; np.ndarray | scipy.sparse.spmatrix:\n        ...\n\n    @overload\n    def get_X(self, dataset_format: Literal[\"dataframe\"]) -&gt; pd.DataFrame:\n        ...\n\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n    ) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        dataset = self.get_dataset()\n        data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n        return data\n\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        # Right now, it is not supported as a feature.\n        # Uncomment if it is supported on the server\n        # in the future.\n        # https://github.com/openml/OpenML/issues/925\n        \"\"\"\n        task_dict = task_container['oml:task_inputs']\n        if self.target_name is not None:\n            task_dict['oml:input'].append(\n                OrderedDict([\n                    ('@name', 'target_feature'),\n                    ('#text', self.target_name)\n                ])\n            )\n        \"\"\"\n        return super()._to_dict()\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLClusteringTask.get_X","title":"<code>get_X(dataset_format='array')</code>","text":"<pre><code>get_X(dataset_format: Literal['array'] = 'array') -&gt; np.ndarray | scipy.sparse.spmatrix\n</code></pre><pre><code>get_X(dataset_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X(\n    self,\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    dataset = self.get_dataset()\n    data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n    return data\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLLearningCurveTask","title":"<code>OpenMLLearningCurveTask</code>","text":"<p>               Bases: <code>OpenMLClassificationTask</code></p> <p>OpenML Learning Curve object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Learning Curve task.</p> required <code>task_type</code> <code>str</code> <p>Name of the Learning Curve task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the dataset that this task is associated with.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature in the dataset.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure to use for evaluating models.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Additional parameters for the estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the file containing the data splits for Learning Curve task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Learning Curve task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure to use for evaluating models.</p> <code>None</code> <code>class_labels</code> <code>list of str</code> <p>Class labels for Learning Curve tasks.</p> <code>None</code> <code>cost_matrix</code> <code>numpy array</code> <p>Cost matrix for Learning Curve tasks.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLLearningCurveTask(OpenMLClassificationTask):\n    \"\"\"OpenML Learning Curve object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Learning Curve task.\n    task_type : str\n        Name of the Learning Curve task.\n    data_set_id : int\n        ID of the dataset that this task is associated with.\n    target_name : str\n        Name of the target feature in the dataset.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure to use for evaluating models.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Additional parameters for the estimation procedure.\n    data_splits_url : str, default=None\n        URL of the file containing the data splits for Learning Curve task.\n    task_id : Union[int, None]\n        ID of the Learning Curve task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure to use for evaluating models.\n    class_labels : list of str, default=None\n        Class labels for Learning Curve tasks.\n    cost_matrix : numpy array, default=None\n        Cost matrix for Learning Curve tasks.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 13,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n            class_labels=class_labels,\n            cost_matrix=cost_matrix,\n        )\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLRegressionTask","title":"<code>OpenMLRegressionTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Regression object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML Regression task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML Regression task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature used in the Regression task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the Regression task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML Regression task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the Regression task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLRegressionTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Regression object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML Regression task.\n    task_type : str\n        Task type of the OpenML Regression task.\n    data_set_id : int\n        ID of the OpenML dataset.\n    target_name : str\n        Name of the target feature used in the Regression task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the Regression task.\n    task_id : Union[int, None]\n        ID of the OpenML Regression task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the Regression task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 7,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSplit","title":"<code>OpenMLSplit</code>","text":"<p>OpenML Split object.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>int or str</code> required <code>description</code> <code>str</code> required <code>split</code> <code>dict</code> required Source code in <code>openml/tasks/split.py</code> <pre><code>class OpenMLSplit:\n    \"\"\"OpenML Split object.\n\n    Parameters\n    ----------\n    name : int or str\n    description : str\n    split : dict\n    \"\"\"\n\n    def __init__(\n        self,\n        name: int | str,\n        description: str,\n        split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]],\n    ):\n        self.description = description\n        self.name = name\n        self.split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]] = {}\n\n        # Add splits according to repetition\n        for repetition in split:\n            _rep = int(repetition)\n            self.split[_rep] = OrderedDict()\n            for fold in split[_rep]:\n                self.split[_rep][fold] = OrderedDict()\n                for sample in split[_rep][fold]:\n                    self.split[_rep][fold][sample] = split[_rep][fold][sample]\n\n        self.repeats = len(self.split)\n\n        # TODO(eddiebergman): Better error message\n        if any(len(self.split[0]) != len(self.split[i]) for i in range(self.repeats)):\n            raise ValueError(\"\")\n\n        self.folds = len(self.split[0])\n        self.samples = len(self.split[0][0])\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if (\n            (not isinstance(self, type(other)))\n            or self.name != other.name\n            or self.description != other.description\n            or self.split.keys() != other.split.keys()\n            or any(\n                self.split[repetition].keys() != other.split[repetition].keys()\n                for repetition in self.split\n            )\n        ):\n            return False\n\n        samples = [\n            (repetition, fold, sample)\n            for repetition in self.split\n            for fold in self.split[repetition]\n            for sample in self.split[repetition][fold]\n        ]\n\n        for repetition, fold, sample in samples:\n            self_train, self_test = self.split[repetition][fold][sample]\n            other_train, other_test = other.split[repetition][fold][sample]\n            if not (np.all(self_train == other_train) and np.all(self_test == other_test)):\n                return False\n        return True\n\n    @classmethod\n    def _from_arff_file(cls, filename: Path) -&gt; OpenMLSplit:  # noqa: C901, PLR0912\n        repetitions = None\n        name = None\n\n        pkl_filename = filename.with_suffix(\".pkl.py3\")\n\n        if pkl_filename.exists():\n            with pkl_filename.open(\"rb\") as fh:\n                # TODO(eddiebergman): Would be good to figure out what _split is and assert it is\n                _split = pickle.load(fh)  # noqa: S301\n            repetitions = _split[\"repetitions\"]\n            name = _split[\"name\"]\n\n        # Cache miss\n        if repetitions is None:\n            # Faster than liac-arff and sufficient in this situation!\n            if not filename.exists():\n                raise FileNotFoundError(f\"Split arff {filename} does not exist!\")\n\n            file_data = arff.load(filename.open(\"r\"), return_type=arff.DENSE_GEN)\n            splits = file_data[\"data\"]\n            name = file_data[\"relation\"]\n            attrnames = [attr[0] for attr in file_data[\"attributes\"]]\n\n            repetitions = OrderedDict()\n\n            type_idx = attrnames.index(\"type\")\n            rowid_idx = attrnames.index(\"rowid\")\n            repeat_idx = attrnames.index(\"repeat\")\n            fold_idx = attrnames.index(\"fold\")\n            sample_idx = attrnames.index(\"sample\") if \"sample\" in attrnames else None\n\n            for line in splits:\n                # A line looks like type, rowid, repeat, fold\n                repetition = int(line[repeat_idx])\n                fold = int(line[fold_idx])\n                sample = 0\n                if sample_idx is not None:\n                    sample = int(line[sample_idx])\n\n                if repetition not in repetitions:\n                    repetitions[repetition] = OrderedDict()\n                if fold not in repetitions[repetition]:\n                    repetitions[repetition][fold] = OrderedDict()\n                if sample not in repetitions[repetition][fold]:\n                    repetitions[repetition][fold][sample] = ([], [])\n                split = repetitions[repetition][fold][sample]\n\n                type_ = line[type_idx]\n                if type_ == \"TRAIN\":\n                    split[0].append(line[rowid_idx])\n                elif type_ == \"TEST\":\n                    split[1].append(line[rowid_idx])\n                else:\n                    raise ValueError(type_)\n\n            for repetition in repetitions:\n                for fold in repetitions[repetition]:\n                    for sample in repetitions[repetition][fold]:\n                        repetitions[repetition][fold][sample] = Split(\n                            np.array(repetitions[repetition][fold][sample][0], dtype=np.int32),\n                            np.array(repetitions[repetition][fold][sample][1], dtype=np.int32),\n                        )\n\n            with pkl_filename.open(\"wb\") as fh:\n                pickle.dump({\"name\": name, \"repetitions\": repetitions}, fh, protocol=2)\n\n        assert name is not None\n        return cls(name, \"\", repetitions)\n\n    def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n        Parameters\n        ----------\n        repeat : int\n            Index of the repeat to retrieve.\n        fold : int\n            Index of the fold to retrieve.\n        sample : int\n            Index of the sample to retrieve.\n\n        Returns\n        -------\n        numpy.ndarray\n            The data split for the specified repeat, fold, and sample.\n\n        Raises\n        ------\n        ValueError\n            If the specified repeat, fold, or sample is not known.\n        \"\"\"\n        if repeat not in self.split:\n            raise ValueError(\"Repeat %s not known\" % str(repeat))\n        if fold not in self.split[repeat]:\n            raise ValueError(\"Fold %s not known\" % str(fold))\n        if sample not in self.split[repeat][fold]:\n            raise ValueError(\"Sample %s not known\" % str(sample))\n        return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSplit.get","title":"<code>get(repeat=0, fold=0, sample=0)</code>","text":"<p>Returns the specified data split from the CrossValidationSplit object.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>Index of the repeat to retrieve.</p> <code>0</code> <code>fold</code> <code>int</code> <p>Index of the fold to retrieve.</p> <code>0</code> <code>sample</code> <code>int</code> <p>Index of the sample to retrieve.</p> <code>0</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>The data split for the specified repeat, fold, and sample.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the specified repeat, fold, or sample is not known.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n    Parameters\n    ----------\n    repeat : int\n        Index of the repeat to retrieve.\n    fold : int\n        Index of the fold to retrieve.\n    sample : int\n        Index of the sample to retrieve.\n\n    Returns\n    -------\n    numpy.ndarray\n        The data split for the specified repeat, fold, and sample.\n\n    Raises\n    ------\n    ValueError\n        If the specified repeat, fold, or sample is not known.\n    \"\"\"\n    if repeat not in self.split:\n        raise ValueError(\"Repeat %s not known\" % str(repeat))\n    if fold not in self.split[repeat]:\n        raise ValueError(\"Fold %s not known\" % str(fold))\n    if sample not in self.split[repeat][fold]:\n        raise ValueError(\"Sample %s not known\" % str(sample))\n    return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSupervisedTask","title":"<code>OpenMLSupervisedTask</code>","text":"<p>               Bases: <code>OpenMLTask</code>, <code>ABC</code></p> <p>OpenML Supervised Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature (the class variable).</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure for the task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure for the task.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the task.</p> <code>None</code> <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLSupervisedTask(OpenMLTask, ABC):\n    \"\"\"OpenML Supervised Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the task type.\n    task_type : str\n        Name of the task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the task.\n    target_name : str\n        Name of the target feature (the class variable).\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure for the task.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure for the task.\n    data_splits_url : str, default=None\n        URL of the data splits for the task.\n    task_id: Union[int, None]\n        Refers to the unique identifier of task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | scipy.sparse.spmatrix,\n        np.ndarray | None,\n    ]:\n        ...\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\"]\n    ) -&gt; tuple[\n        pd.DataFrame,\n        pd.Series | pd.DataFrame | None,\n    ]:\n        ...\n\n    # TODO(eddiebergman): Do all OpenMLSupervisedTask have a `y`?\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n        np.ndarray | pd.Series | pd.DataFrame | None,\n    ]:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        dataset = self.get_dataset()\n        if self.task_type_id not in (\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.SUPERVISED_REGRESSION,\n            TaskType.LEARNING_CURVE,\n        ):\n            raise NotImplementedError(self.task_type)\n\n        X, y, _, _ = dataset.get_data(\n            dataset_format=dataset_format,\n            target=self.target_name,\n        )\n        return X, y\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        task_container = super()._to_dict()\n        oml_input = task_container[\"oml:task_inputs\"][\"oml:input\"]  # type: ignore\n        assert isinstance(oml_input, list)\n\n        oml_input.append({\"@name\": \"target_feature\", \"#text\": self.target_name})\n        return task_container\n\n    @property\n    def estimation_parameters(self) -&gt; dict[str, str] | None:\n        \"\"\"Return the estimation parameters for the task.\"\"\"\n        warnings.warn(\n            \"The estimation_parameters attribute will be \"\n            \"deprecated in the future, please use \"\n            \"estimation_procedure['parameters'] instead\",\n            PendingDeprecationWarning,\n            stacklevel=2,\n        )\n        return self.estimation_procedure[\"parameters\"]\n\n    @estimation_parameters.setter\n    def estimation_parameters(self, est_parameters: dict[str, str] | None) -&gt; None:\n        self.estimation_procedure[\"parameters\"] = est_parameters\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSupervisedTask.estimation_parameters","title":"<code>estimation_parameters: dict[str, str] | None</code>  <code>property</code> <code>writable</code>","text":"<p>Return the estimation parameters for the task.</p>"},{"location":"reference/tasks/#openml.tasks.OpenMLSupervisedTask.get_X_and_y","title":"<code>get_X_and_y(dataset_format='array')</code>","text":"<pre><code>get_X_and_y(dataset_format: Literal['array'] = 'array') -&gt; tuple[np.ndarray | scipy.sparse.spmatrix, np.ndarray | None]\n</code></pre><pre><code>get_X_and_y(dataset_format: Literal['dataframe']) -&gt; tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X_and_y(\n    self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n    np.ndarray | pd.Series | pd.DataFrame | None,\n]:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    dataset = self.get_dataset()\n    if self.task_type_id not in (\n        TaskType.SUPERVISED_CLASSIFICATION,\n        TaskType.SUPERVISED_REGRESSION,\n        TaskType.LEARNING_CURVE,\n    ):\n        raise NotImplementedError(self.task_type)\n\n    X, y, _, _ = dataset.get_data(\n        dataset_format=dataset_format,\n        target=self.target_name,\n    )\n    return X, y\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask","title":"<code>OpenMLTask</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Task object.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of OpenML task.</p> required <code>task_type_id</code> <code>TaskType</code> <p>Refers to the type of OpenML task.</p> required <code>task_type</code> <code>str</code> <p>Refers to the OpenML task.</p> required <code>data_set_id</code> <code>int</code> <p>Refers to the data.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>Refers to the type of estimates used.</p> <code>1</code> <code>estimation_procedure_type</code> <code>str | None</code> <p>Refers to the type of estimation procedure used for the OpenML task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict[str, str] | None</code> <p>Estimation parameters used for the OpenML task.</p> <code>None</code> <code>evaluation_measure</code> <code>str | None</code> <p>Refers to the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str | None</code> <p>Refers to the URL of the data splits used for the OpenML task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLTask(OpenMLBase):\n    \"\"\"OpenML Task object.\n\n    Parameters\n    ----------\n    task_id: Union[int, None]\n        Refers to the unique identifier of OpenML task.\n    task_type_id: TaskType\n        Refers to the type of OpenML task.\n    task_type: str\n        Refers to the OpenML task.\n    data_set_id: int\n        Refers to the data.\n    estimation_procedure_id: int\n        Refers to the type of estimates used.\n    estimation_procedure_type: str, default=None\n        Refers to the type of estimation procedure used for the OpenML task.\n    estimation_parameters: [Dict[str, str]], default=None\n        Estimation parameters used for the OpenML task.\n    evaluation_measure: str, default=None\n        Refers to the evaluation measure.\n    data_splits_url: str, default=None\n        Refers to the URL of the data splits used for the OpenML task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int | None,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n    ):\n        self.task_id = int(task_id) if task_id is not None else None\n        self.task_type_id = task_type_id\n        self.task_type = task_type\n        self.dataset_id = int(data_set_id)\n        self.evaluation_measure = evaluation_measure\n        self.estimation_procedure: _EstimationProcedure = {\n            \"type\": estimation_procedure_type,\n            \"parameters\": estimation_parameters,\n            \"data_splits_url\": data_splits_url,\n        }\n        self.estimation_procedure_id = estimation_procedure_id\n        self.split: OpenMLSplit | None = None\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"t\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the OpenML ID of this task.\"\"\"\n        return self.task_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        base_server_url = openml.config.get_server_base_url()\n        fields: dict[str, Any] = {\n            \"Task Type Description\": f\"{base_server_url}/tt/{self.task_type_id}\"\n        }\n        if self.task_id is not None:\n            fields[\"Task ID\"] = self.task_id\n            fields[\"Task URL\"] = self.openml_url\n        if self.evaluation_measure is not None:\n            fields[\"Evaluation Measure\"] = self.evaluation_measure\n        if self.estimation_procedure is not None:\n            fields[\"Estimation Procedure\"] = self.estimation_procedure[\"type\"]\n\n        # TODO(eddiebergman): Subclasses could advertise/provide this, instead of having to\n        # have the base class know about it's subclasses.\n        target_name = getattr(self, \"target_name\", None)\n        if target_name is not None:\n            fields[\"Target Feature\"] = target_name\n\n            class_labels = getattr(self, \"class_labels\", None)\n            if class_labels is not None:\n                fields[\"# of Classes\"] = len(class_labels)\n\n            if hasattr(self, \"cost_matrix\"):\n                fields[\"Cost Matrix\"] = \"Available\"\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Task Type Description\",\n            \"Task ID\",\n            \"Task URL\",\n            \"Estimation Procedure\",\n            \"Evaluation Measure\",\n            \"Target Feature\",\n            \"# of Classes\",\n            \"Cost Matrix\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def get_dataset(self) -&gt; datasets.OpenMLDataset:\n        \"\"\"Download dataset associated with task.\"\"\"\n        return datasets.get_dataset(self.dataset_id)\n\n    def get_train_test_split_indices(\n        self,\n        fold: int = 0,\n        repeat: int = 0,\n        sample: int = 0,\n    ) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n        # Replace with retrieve from cache\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.get(repeat=repeat, fold=fold, sample=sample)\n\n    def _download_split(self, cache_file: Path) -&gt; None:\n        # TODO(eddiebergman): Not sure about this try to read and error approach\n        try:\n            with cache_file.open(encoding=\"utf8\"):\n                pass\n        except OSError:\n            split_url = self.estimation_procedure[\"data_splits_url\"]\n            openml._api_calls._download_text_file(\n                source=str(split_url),\n                output_path=str(cache_file),\n            )\n\n    def download_split(self) -&gt; OpenMLSplit:\n        \"\"\"Download the OpenML split for a given task.\"\"\"\n        # TODO(eddiebergman): Can this every be `None`?\n        assert self.task_id is not None\n        cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n        cached_split_file = cache_dir / \"datasplits.arff\"\n\n        try:\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n        except OSError:\n            # Next, download and cache the associated split file\n            self._download_split(cached_split_file)\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n\n        return split\n\n    def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n        \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.repeats, self.split.folds, self.split.samples\n\n    # TODO(eddiebergman): Really need some better typing on all this\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        \"\"\"Creates a dictionary representation of self in a string format (for XML parsing).\"\"\"\n        oml_input = [\n            {\"@name\": \"source_data\", \"#text\": str(self.dataset_id)},\n            {\"@name\": \"estimation_procedure\", \"#text\": str(self.estimation_procedure_id)},\n        ]\n        if self.evaluation_measure is not None:  #\n            oml_input.append({\"@name\": \"evaluation_measures\", \"#text\": self.evaluation_measure})\n\n        return {\n            \"oml:task_inputs\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                \"oml:task_type_id\": self.task_type_id.value,  # This is an int from the enum?\n                \"oml:input\": oml_input,\n            }\n        }\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.task_id = int(xml_response[\"oml:upload_task\"][\"oml:id\"])\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the OpenML ID of this task.</p>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.download_split","title":"<code>download_split()</code>","text":"<p>Download the OpenML split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def download_split(self) -&gt; OpenMLSplit:\n    \"\"\"Download the OpenML split for a given task.\"\"\"\n    # TODO(eddiebergman): Can this every be `None`?\n    assert self.task_id is not None\n    cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n    cached_split_file = cache_dir / \"datasplits.arff\"\n\n    try:\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n    except OSError:\n        # Next, download and cache the associated split file\n        self._download_split(cached_split_file)\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n\n    return split\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.get_dataset","title":"<code>get_dataset()</code>","text":"<p>Download dataset associated with task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_dataset(self) -&gt; datasets.OpenMLDataset:\n    \"\"\"Download dataset associated with task.\"\"\"\n    return datasets.get_dataset(self.dataset_id)\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.get_split_dimensions","title":"<code>get_split_dimensions()</code>","text":"<p>Get the (repeats, folds, samples) of the split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n    \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.repeats, self.split.folds, self.split.samples\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.get_train_test_split_indices","title":"<code>get_train_test_split_indices(fold=0, repeat=0, sample=0)</code>","text":"<p>Get the indices of the train and test splits for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_train_test_split_indices(\n    self,\n    fold: int = 0,\n    repeat: int = 0,\n    sample: int = 0,\n) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n    # Replace with retrieve from cache\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.get(repeat=repeat, fold=fold, sample=sample)\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.TaskType","title":"<code>TaskType</code>","text":"<p>               Bases: <code>Enum</code></p> <p>Possible task types as defined in OpenML.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>class TaskType(Enum):\n    \"\"\"Possible task types as defined in OpenML.\"\"\"\n\n    SUPERVISED_CLASSIFICATION = 1\n    SUPERVISED_REGRESSION = 2\n    LEARNING_CURVE = 3\n    SUPERVISED_DATASTREAM_CLASSIFICATION = 4\n    CLUSTERING = 5\n    MACHINE_LEARNING_CHALLENGE = 6\n    SURVIVAL_ANALYSIS = 7\n    SUBGROUP_DISCOVERY = 8\n    MULTITASK_REGRESSION = 9\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.create_task","title":"<code>create_task(task_type, dataset_id, estimation_procedure_id, target_name=None, evaluation_measure=None, **kwargs)</code>","text":"<p>Create a task based on different given attributes.</p> <p>Builds a task object with the function arguments as attributes. The type of the task object built is determined from the task type id. More information on how the arguments (task attributes), relate to the different possible tasks can be found in the individual task objects at the openml.tasks.task module.</p> <p>Parameters:</p> Name Type Description Default <code>task_type</code> <code>TaskType</code> <p>Id of the task type.</p> required <code>dataset_id</code> <code>int</code> <p>The id of the dataset for the task.</p> required <code>target_name</code> <code>str</code> <p>The name of the feature used as a target. At the moment, only optional for the clustering tasks.</p> <code>None</code> <code>estimation_procedure_id</code> <code>int</code> <p>The id of the estimation procedure.</p> required <code>evaluation_measure</code> <code>str</code> <p>The name of the evaluation measure.</p> <code>None</code> <code>kwargs</code> <code>dict</code> <p>Other task attributes that are not mandatory for task upload.</p> <code>{}</code> <p>Returns:</p> Type Description <code>(OpenMLClassificationTask, OpenMLRegressionTask)</code> <code>(OpenMLLearningCurveTask, OpenMLClusteringTask)</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def create_task(\n    task_type: TaskType,\n    dataset_id: int,\n    estimation_procedure_id: int,\n    target_name: str | None = None,\n    evaluation_measure: str | None = None,\n    **kwargs: Any,\n) -&gt; (\n    OpenMLClassificationTask | OpenMLRegressionTask | OpenMLLearningCurveTask | OpenMLClusteringTask\n):\n    \"\"\"Create a task based on different given attributes.\n\n    Builds a task object with the function arguments as\n    attributes. The type of the task object built is\n    determined from the task type id.\n    More information on how the arguments (task attributes),\n    relate to the different possible tasks can be found in\n    the individual task objects at the openml.tasks.task\n    module.\n\n    Parameters\n    ----------\n    task_type : TaskType\n        Id of the task type.\n    dataset_id : int\n        The id of the dataset for the task.\n    target_name : str, optional\n        The name of the feature used as a target.\n        At the moment, only optional for the clustering tasks.\n    estimation_procedure_id : int\n        The id of the estimation procedure.\n    evaluation_measure : str, optional\n        The name of the evaluation measure.\n    kwargs : dict, optional\n        Other task attributes that are not mandatory\n        for task upload.\n\n    Returns\n    -------\n    OpenMLClassificationTask, OpenMLRegressionTask,\n    OpenMLLearningCurveTask, OpenMLClusteringTask\n    \"\"\"\n    if task_type == TaskType.CLUSTERING:\n        task_cls = OpenMLClusteringTask\n    elif task_type == TaskType.LEARNING_CURVE:\n        task_cls = OpenMLLearningCurveTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_CLASSIFICATION:\n        task_cls = OpenMLClassificationTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_REGRESSION:\n        task_cls = OpenMLRegressionTask  # type: ignore\n    else:\n        raise NotImplementedError(f\"Task type {task_type:d} not supported.\")\n\n    return task_cls(\n        task_type_id=task_type,\n        task_type=\"None\",  # TODO: refactor to get task type string from ID.\n        data_set_id=dataset_id,\n        target_name=target_name,\n        estimation_procedure_id=estimation_procedure_id,\n        evaluation_measure=evaluation_measure,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.delete_task","title":"<code>delete_task(task_id)</code>","text":"<p>Delete task with id <code>task_id</code> from the OpenML server.</p> <p>You can only delete tasks which you created and have no runs associated with them.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>OpenML id of the task</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def delete_task(task_id: int) -&gt; bool:\n    \"\"\"Delete task with id `task_id` from the OpenML server.\n\n    You can only delete tasks which you created and have\n    no runs associated with them.\n\n    Parameters\n    ----------\n    task_id : int\n        OpenML id of the task\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"task\", task_id)\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.get_task","title":"<code>get_task(task_id, *dataset_args, download_splits=None, **get_dataset_kwargs)</code>","text":"<p>Download OpenML task for a given task ID.</p> <p>Downloads the task representation. By default, this will also download the data splits and the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by default.</p> <p>Use the <code>download_splits</code> parameter to control whether the splits are downloaded. Moreover, you may pass additional parameter (args or kwargs) that are passed to :meth:<code>openml.datasets.get_dataset</code>. For backwards compatibility, if <code>download_data</code> is passed as an additional parameter and <code>download_splits</code> is not explicitly set, <code>download_data</code> also overrules <code>download_splits</code>'s value (deprecated from Version 0.15.0 onwards).</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The OpenML task id of the task to download.</p> required <code>download_splits</code> <code>bool | None</code> <p>Whether to download the splits as well. From version 0.15.0 onwards this is independent of download_data and will default to <code>False</code>.</p> <code>None</code> <code>dataset_args</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <code>get_dataset_kwargs</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <p>Returns:</p> Name Type Description <code>task</code> <code>OpenMLTask</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_task(\n    task_id: int,\n    *dataset_args: Any,\n    download_splits: bool | None = None,\n    **get_dataset_kwargs: Any,\n) -&gt; OpenMLTask:\n    \"\"\"Download OpenML task for a given task ID.\n\n    Downloads the task representation. By default, this will also download the data splits and\n    the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by\n    default.\n\n    Use the `download_splits` parameter to control whether the splits are downloaded.\n    Moreover, you may pass additional parameter (args or kwargs) that are passed to\n    :meth:`openml.datasets.get_dataset`.\n    For backwards compatibility, if `download_data` is passed as an additional parameter and\n    `download_splits` is not explicitly set, `download_data` also overrules `download_splits`'s\n    value (deprecated from Version 0.15.0 onwards).\n\n    Parameters\n    ----------\n    task_id : int\n        The OpenML task id of the task to download.\n    download_splits: bool (default=True)\n        Whether to download the splits as well. From version 0.15.0 onwards this is independent\n        of download_data and will default to ``False``.\n    dataset_args, get_dataset_kwargs :\n        Args and kwargs can be used pass optional parameters to :meth:`openml.datasets.get_dataset`.\n        This includes `download_data`. If set to True the splits are downloaded as well\n        (deprecated from Version 0.15.0 onwards). The args are only present for backwards\n        compatibility and will be removed from version 0.15.0 onwards.\n\n    Returns\n    -------\n    task: OpenMLTask\n    \"\"\"\n    if download_splits is None:\n        # TODO(0.15): Switch download splits to False by default, adjust typing above, adjust\n        #  documentation above, and remove warning.\n        warnings.warn(\n            \"Starting from Version 0.15.0 `download_splits` will default to ``False`` instead \"\n            \"of ``True`` and be independent from `download_data`. To disable this message until \"\n            \"version 0.15 explicitly set `download_splits` to a bool.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n        download_splits = get_dataset_kwargs.get(\"download_data\", True)\n\n    if not isinstance(task_id, int):\n        # TODO(0.15): Remove warning\n        warnings.warn(\n            \"Task id must be specified as `int` from 0.14.0 onwards.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n\n    try:\n        task_id = int(task_id)\n    except (ValueError, TypeError) as e:\n        raise ValueError(\"Dataset ID is neither an Integer nor can be cast to an Integer.\") from e\n\n    tid_cache_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id)\n\n    try:\n        task = _get_task_description(task_id)\n        dataset = get_dataset(task.dataset_id, *dataset_args, **get_dataset_kwargs)\n        # List of class labels available in dataset description\n        # Including class labels as part of task meta data handles\n        #   the case where data download was initially disabled\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            task.class_labels = dataset.retrieve_class_labels(task.target_name)\n        # Clustering tasks do not have class labels\n        # and do not offer download_split\n        if download_splits and isinstance(task, OpenMLSupervisedTask):\n            task.download_split()\n    except Exception as e:\n        openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir)\n        raise e\n\n    return task\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.get_tasks","title":"<code>get_tasks(task_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download tasks.</p> <p>This function iterates :meth:<code>openml.tasks.get_task</code>.</p> <p>Parameters:</p> Name Type Description Default <code>task_ids</code> <code>List[int]</code> <p>A list of task ids to download.</p> required <code>download_data</code> <code>bool(default=True)</code> <p>Option to trigger download of data along with the meta data.</p> <code>True</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description.</p> <code>True</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def get_tasks(\n    task_ids: list[int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLTask]:\n    \"\"\"Download tasks.\n\n    This function iterates :meth:`openml.tasks.get_task`.\n\n    Parameters\n    ----------\n    task_ids : List[int]\n        A list of task ids to download.\n    download_data : bool (default = True)\n        Option to trigger download of data along with the meta data.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    tasks = []\n    for task_id in task_ids:\n        tasks.append(get_task(task_id, download_data, download_qualities))\n    return tasks\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.list_tasks","title":"<code>list_tasks(task_type=None, offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<p>Return a number of tasks having the given tag and task_type</p> <p>Parameters:</p> Name Type Description Default <code>Filter</code> required <code>it</code> required <code>type</code> required <code>task_type</code> <code>TaskType</code> <p>Refers to the type of task.</p> <code>None</code> <code>offset</code> <code>int</code> <p>the number of tasks to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of tasks to show</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: data_tag, status, data_id, data_name, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a dictionary containing the following information: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> <code>dataframe</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a row in the data frame containing the following information as columns: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def list_tasks(\n    task_type: TaskType | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a number of tasks having the given tag and task_type\n\n    Parameters\n    ----------\n    Filter task_type is separated from the other filters because\n    it is used as task_type in the task description, but it is named\n    type when used as a filter in list tasks call.\n    task_type : TaskType, optional\n        Refers to the type of task.\n    offset : int, optional\n        the number of tasks to skip, starting from the first\n    size : int, optional\n        the maximum number of tasks to show\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: data_tag, status, data_id, data_name,\n        number_instances, number_features,\n        number_classes, number_missing_values.\n\n    Returns\n    -------\n    dict\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a dictionary containing the following information:\n        task id, dataset id, task_type and status. If qualities are calculated\n        for the associated dataset, some of these are also returned.\n    dataframe\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a row in the data frame containing the following information\n        as columns: task id, dataset id, task_type and status. If qualities are\n        calculated for the associated dataset, some of these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_tasks,\n        task_type=task_type,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/functions/","title":"functions","text":""},{"location":"reference/tasks/functions/#openml.tasks.functions.__list_tasks","title":"<code>__list_tasks(api_call, output_format='dict')</code>","text":"<p>Returns a dictionary or a Pandas DataFrame with information about OpenML tasks.</p> <p>Parameters:</p> Name Type Description Default <code>api_call</code> <code>str</code> <p>The API call specifying which tasks to return.</p> required <code>output_format</code> <code>str in {'dict', 'dataframe'}</code> <p>Output format for the returned object.</p> <code>'dict'</code> <p>Returns:</p> Type Description <code>Union[Dict, DataFrame]</code> <p>A dictionary or a Pandas DataFrame with information about OpenML tasks.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the XML returned by the OpenML API does not contain 'oml:tasks', '@xmlns:oml', or has an incorrect value for '@xmlns:oml'.</p> <code>KeyError</code> <p>If an invalid key is found in the XML for a task.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def __list_tasks(  # noqa: PLR0912, C901\n    api_call: str,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Returns a dictionary or a Pandas DataFrame with information about OpenML tasks.\n\n    Parameters\n    ----------\n    api_call : str\n        The API call specifying which tasks to return.\n    output_format : str in {\"dict\", \"dataframe\"}\n        Output format for the returned object.\n\n    Returns\n    -------\n    Union[Dict, pd.DataFrame]\n        A dictionary or a Pandas DataFrame with information about OpenML tasks.\n\n    Raises\n    ------\n    ValueError\n        If the XML returned by the OpenML API does not contain 'oml:tasks', '@xmlns:oml',\n        or has an incorrect value for '@xmlns:oml'.\n    KeyError\n        If an invalid key is found in the XML for a task.\n    \"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    tasks_dict = xmltodict.parse(xml_string, force_list=(\"oml:task\", \"oml:input\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:tasks\" not in tasks_dict:\n        raise ValueError(f'Error in return XML, does not contain \"oml:runs\": {tasks_dict}')\n\n    if \"@xmlns:oml\" not in tasks_dict[\"oml:tasks\"]:\n        raise ValueError(\n            f'Error in return XML, does not contain \"oml:runs\"/@xmlns:oml: {tasks_dict}'\n        )\n\n    if tasks_dict[\"oml:tasks\"][\"@xmlns:oml\"] != \"http://openml.org/openml\":\n        raise ValueError(\n            \"Error in return XML, value of  \"\n            '\"oml:runs\"/@xmlns:oml is not '\n            '\"http://openml.org/openml\": %s' % str(tasks_dict),\n        )\n\n    assert isinstance(tasks_dict[\"oml:tasks\"][\"oml:task\"], list), type(tasks_dict[\"oml:tasks\"])\n\n    tasks = {}\n    procs = _get_estimation_procedure_list()\n    proc_dict = {x[\"id\"]: x for x in procs}\n\n    for task_ in tasks_dict[\"oml:tasks\"][\"oml:task\"]:\n        tid = None\n        try:\n            tid = int(task_[\"oml:task_id\"])\n            task_type_int = int(task_[\"oml:task_type_id\"])\n            try:\n                task_type_id = TaskType(task_type_int)\n            except ValueError as e:\n                warnings.warn(\n                    f\"Could not create task type id for {task_type_int} due to error {e}\",\n                    RuntimeWarning,\n                    stacklevel=2,\n                )\n                continue\n\n            task = {\n                \"tid\": tid,\n                \"ttid\": task_type_id,\n                \"did\": int(task_[\"oml:did\"]),\n                \"name\": task_[\"oml:name\"],\n                \"task_type\": task_[\"oml:task_type\"],\n                \"status\": task_[\"oml:status\"],\n            }\n\n            # Other task inputs\n            for _input in task_.get(\"oml:input\", []):\n                if _input[\"@name\"] == \"estimation_procedure\":\n                    task[_input[\"@name\"]] = proc_dict[int(_input[\"#text\"])][\"name\"]\n                else:\n                    value = _input.get(\"#text\")\n                    task[_input[\"@name\"]] = value\n\n            # The number of qualities can range from 0 to infinity\n            for quality in task_.get(\"oml:quality\", []):\n                if \"#text\" not in quality:\n                    quality_value = 0.0\n                else:\n                    quality[\"#text\"] = float(quality[\"#text\"])\n                    if abs(int(quality[\"#text\"]) - quality[\"#text\"]) &lt; 0.0000001:\n                        quality[\"#text\"] = int(quality[\"#text\"])\n                    quality_value = quality[\"#text\"]\n                task[quality[\"@name\"]] = quality_value\n            tasks[tid] = task\n        except KeyError as e:\n            if tid is not None:\n                warnings.warn(\n                    \"Invalid xml for task %d: %s\\nFrom %s\" % (tid, e, task_),\n                    RuntimeWarning,\n                    stacklevel=2,\n                )\n            else:\n                warnings.warn(f\"Could not find key {e} in {task_}!\", RuntimeWarning, stacklevel=2)\n\n    if output_format == \"dataframe\":\n        tasks = pd.DataFrame.from_dict(tasks, orient=\"index\")\n\n    return tasks\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.create_task","title":"<code>create_task(task_type, dataset_id, estimation_procedure_id, target_name=None, evaluation_measure=None, **kwargs)</code>","text":"<p>Create a task based on different given attributes.</p> <p>Builds a task object with the function arguments as attributes. The type of the task object built is determined from the task type id. More information on how the arguments (task attributes), relate to the different possible tasks can be found in the individual task objects at the openml.tasks.task module.</p> <p>Parameters:</p> Name Type Description Default <code>task_type</code> <code>TaskType</code> <p>Id of the task type.</p> required <code>dataset_id</code> <code>int</code> <p>The id of the dataset for the task.</p> required <code>target_name</code> <code>str</code> <p>The name of the feature used as a target. At the moment, only optional for the clustering tasks.</p> <code>None</code> <code>estimation_procedure_id</code> <code>int</code> <p>The id of the estimation procedure.</p> required <code>evaluation_measure</code> <code>str</code> <p>The name of the evaluation measure.</p> <code>None</code> <code>kwargs</code> <code>dict</code> <p>Other task attributes that are not mandatory for task upload.</p> <code>{}</code> <p>Returns:</p> Type Description <code>(OpenMLClassificationTask, OpenMLRegressionTask)</code> <code>(OpenMLLearningCurveTask, OpenMLClusteringTask)</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def create_task(\n    task_type: TaskType,\n    dataset_id: int,\n    estimation_procedure_id: int,\n    target_name: str | None = None,\n    evaluation_measure: str | None = None,\n    **kwargs: Any,\n) -&gt; (\n    OpenMLClassificationTask | OpenMLRegressionTask | OpenMLLearningCurveTask | OpenMLClusteringTask\n):\n    \"\"\"Create a task based on different given attributes.\n\n    Builds a task object with the function arguments as\n    attributes. The type of the task object built is\n    determined from the task type id.\n    More information on how the arguments (task attributes),\n    relate to the different possible tasks can be found in\n    the individual task objects at the openml.tasks.task\n    module.\n\n    Parameters\n    ----------\n    task_type : TaskType\n        Id of the task type.\n    dataset_id : int\n        The id of the dataset for the task.\n    target_name : str, optional\n        The name of the feature used as a target.\n        At the moment, only optional for the clustering tasks.\n    estimation_procedure_id : int\n        The id of the estimation procedure.\n    evaluation_measure : str, optional\n        The name of the evaluation measure.\n    kwargs : dict, optional\n        Other task attributes that are not mandatory\n        for task upload.\n\n    Returns\n    -------\n    OpenMLClassificationTask, OpenMLRegressionTask,\n    OpenMLLearningCurveTask, OpenMLClusteringTask\n    \"\"\"\n    if task_type == TaskType.CLUSTERING:\n        task_cls = OpenMLClusteringTask\n    elif task_type == TaskType.LEARNING_CURVE:\n        task_cls = OpenMLLearningCurveTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_CLASSIFICATION:\n        task_cls = OpenMLClassificationTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_REGRESSION:\n        task_cls = OpenMLRegressionTask  # type: ignore\n    else:\n        raise NotImplementedError(f\"Task type {task_type:d} not supported.\")\n\n    return task_cls(\n        task_type_id=task_type,\n        task_type=\"None\",  # TODO: refactor to get task type string from ID.\n        data_set_id=dataset_id,\n        target_name=target_name,\n        estimation_procedure_id=estimation_procedure_id,\n        evaluation_measure=evaluation_measure,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.delete_task","title":"<code>delete_task(task_id)</code>","text":"<p>Delete task with id <code>task_id</code> from the OpenML server.</p> <p>You can only delete tasks which you created and have no runs associated with them.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>OpenML id of the task</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def delete_task(task_id: int) -&gt; bool:\n    \"\"\"Delete task with id `task_id` from the OpenML server.\n\n    You can only delete tasks which you created and have\n    no runs associated with them.\n\n    Parameters\n    ----------\n    task_id : int\n        OpenML id of the task\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"task\", task_id)\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.get_task","title":"<code>get_task(task_id, *dataset_args, download_splits=None, **get_dataset_kwargs)</code>","text":"<p>Download OpenML task for a given task ID.</p> <p>Downloads the task representation. By default, this will also download the data splits and the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by default.</p> <p>Use the <code>download_splits</code> parameter to control whether the splits are downloaded. Moreover, you may pass additional parameter (args or kwargs) that are passed to :meth:<code>openml.datasets.get_dataset</code>. For backwards compatibility, if <code>download_data</code> is passed as an additional parameter and <code>download_splits</code> is not explicitly set, <code>download_data</code> also overrules <code>download_splits</code>'s value (deprecated from Version 0.15.0 onwards).</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The OpenML task id of the task to download.</p> required <code>download_splits</code> <code>bool | None</code> <p>Whether to download the splits as well. From version 0.15.0 onwards this is independent of download_data and will default to <code>False</code>.</p> <code>None</code> <code>dataset_args</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <code>get_dataset_kwargs</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <p>Returns:</p> Name Type Description <code>task</code> <code>OpenMLTask</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_task(\n    task_id: int,\n    *dataset_args: Any,\n    download_splits: bool | None = None,\n    **get_dataset_kwargs: Any,\n) -&gt; OpenMLTask:\n    \"\"\"Download OpenML task for a given task ID.\n\n    Downloads the task representation. By default, this will also download the data splits and\n    the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by\n    default.\n\n    Use the `download_splits` parameter to control whether the splits are downloaded.\n    Moreover, you may pass additional parameter (args or kwargs) that are passed to\n    :meth:`openml.datasets.get_dataset`.\n    For backwards compatibility, if `download_data` is passed as an additional parameter and\n    `download_splits` is not explicitly set, `download_data` also overrules `download_splits`'s\n    value (deprecated from Version 0.15.0 onwards).\n\n    Parameters\n    ----------\n    task_id : int\n        The OpenML task id of the task to download.\n    download_splits: bool (default=True)\n        Whether to download the splits as well. From version 0.15.0 onwards this is independent\n        of download_data and will default to ``False``.\n    dataset_args, get_dataset_kwargs :\n        Args and kwargs can be used pass optional parameters to :meth:`openml.datasets.get_dataset`.\n        This includes `download_data`. If set to True the splits are downloaded as well\n        (deprecated from Version 0.15.0 onwards). The args are only present for backwards\n        compatibility and will be removed from version 0.15.0 onwards.\n\n    Returns\n    -------\n    task: OpenMLTask\n    \"\"\"\n    if download_splits is None:\n        # TODO(0.15): Switch download splits to False by default, adjust typing above, adjust\n        #  documentation above, and remove warning.\n        warnings.warn(\n            \"Starting from Version 0.15.0 `download_splits` will default to ``False`` instead \"\n            \"of ``True`` and be independent from `download_data`. To disable this message until \"\n            \"version 0.15 explicitly set `download_splits` to a bool.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n        download_splits = get_dataset_kwargs.get(\"download_data\", True)\n\n    if not isinstance(task_id, int):\n        # TODO(0.15): Remove warning\n        warnings.warn(\n            \"Task id must be specified as `int` from 0.14.0 onwards.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n\n    try:\n        task_id = int(task_id)\n    except (ValueError, TypeError) as e:\n        raise ValueError(\"Dataset ID is neither an Integer nor can be cast to an Integer.\") from e\n\n    tid_cache_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id)\n\n    try:\n        task = _get_task_description(task_id)\n        dataset = get_dataset(task.dataset_id, *dataset_args, **get_dataset_kwargs)\n        # List of class labels available in dataset description\n        # Including class labels as part of task meta data handles\n        #   the case where data download was initially disabled\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            task.class_labels = dataset.retrieve_class_labels(task.target_name)\n        # Clustering tasks do not have class labels\n        # and do not offer download_split\n        if download_splits and isinstance(task, OpenMLSupervisedTask):\n            task.download_split()\n    except Exception as e:\n        openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir)\n        raise e\n\n    return task\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.get_tasks","title":"<code>get_tasks(task_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download tasks.</p> <p>This function iterates :meth:<code>openml.tasks.get_task</code>.</p> <p>Parameters:</p> Name Type Description Default <code>task_ids</code> <code>List[int]</code> <p>A list of task ids to download.</p> required <code>download_data</code> <code>bool(default=True)</code> <p>Option to trigger download of data along with the meta data.</p> <code>True</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description.</p> <code>True</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def get_tasks(\n    task_ids: list[int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLTask]:\n    \"\"\"Download tasks.\n\n    This function iterates :meth:`openml.tasks.get_task`.\n\n    Parameters\n    ----------\n    task_ids : List[int]\n        A list of task ids to download.\n    download_data : bool (default = True)\n        Option to trigger download of data along with the meta data.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    tasks = []\n    for task_id in task_ids:\n        tasks.append(get_task(task_id, download_data, download_qualities))\n    return tasks\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.list_tasks","title":"<code>list_tasks(task_type=None, offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<p>Return a number of tasks having the given tag and task_type</p> <p>Parameters:</p> Name Type Description Default <code>Filter</code> required <code>it</code> required <code>type</code> required <code>task_type</code> <code>TaskType</code> <p>Refers to the type of task.</p> <code>None</code> <code>offset</code> <code>int</code> <p>the number of tasks to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of tasks to show</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: data_tag, status, data_id, data_name, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a dictionary containing the following information: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> <code>dataframe</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a row in the data frame containing the following information as columns: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def list_tasks(\n    task_type: TaskType | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a number of tasks having the given tag and task_type\n\n    Parameters\n    ----------\n    Filter task_type is separated from the other filters because\n    it is used as task_type in the task description, but it is named\n    type when used as a filter in list tasks call.\n    task_type : TaskType, optional\n        Refers to the type of task.\n    offset : int, optional\n        the number of tasks to skip, starting from the first\n    size : int, optional\n        the maximum number of tasks to show\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: data_tag, status, data_id, data_name,\n        number_instances, number_features,\n        number_classes, number_missing_values.\n\n    Returns\n    -------\n    dict\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a dictionary containing the following information:\n        task id, dataset id, task_type and status. If qualities are calculated\n        for the associated dataset, some of these are also returned.\n    dataframe\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a row in the data frame containing the following information\n        as columns: task id, dataset id, task_type and status. If qualities are\n        calculated for the associated dataset, some of these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_tasks,\n        task_type=task_type,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/split/","title":"split","text":""},{"location":"reference/tasks/split/#openml.tasks.split.OpenMLSplit","title":"<code>OpenMLSplit</code>","text":"<p>OpenML Split object.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>int or str</code> required <code>description</code> <code>str</code> required <code>split</code> <code>dict</code> required Source code in <code>openml/tasks/split.py</code> <pre><code>class OpenMLSplit:\n    \"\"\"OpenML Split object.\n\n    Parameters\n    ----------\n    name : int or str\n    description : str\n    split : dict\n    \"\"\"\n\n    def __init__(\n        self,\n        name: int | str,\n        description: str,\n        split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]],\n    ):\n        self.description = description\n        self.name = name\n        self.split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]] = {}\n\n        # Add splits according to repetition\n        for repetition in split:\n            _rep = int(repetition)\n            self.split[_rep] = OrderedDict()\n            for fold in split[_rep]:\n                self.split[_rep][fold] = OrderedDict()\n                for sample in split[_rep][fold]:\n                    self.split[_rep][fold][sample] = split[_rep][fold][sample]\n\n        self.repeats = len(self.split)\n\n        # TODO(eddiebergman): Better error message\n        if any(len(self.split[0]) != len(self.split[i]) for i in range(self.repeats)):\n            raise ValueError(\"\")\n\n        self.folds = len(self.split[0])\n        self.samples = len(self.split[0][0])\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if (\n            (not isinstance(self, type(other)))\n            or self.name != other.name\n            or self.description != other.description\n            or self.split.keys() != other.split.keys()\n            or any(\n                self.split[repetition].keys() != other.split[repetition].keys()\n                for repetition in self.split\n            )\n        ):\n            return False\n\n        samples = [\n            (repetition, fold, sample)\n            for repetition in self.split\n            for fold in self.split[repetition]\n            for sample in self.split[repetition][fold]\n        ]\n\n        for repetition, fold, sample in samples:\n            self_train, self_test = self.split[repetition][fold][sample]\n            other_train, other_test = other.split[repetition][fold][sample]\n            if not (np.all(self_train == other_train) and np.all(self_test == other_test)):\n                return False\n        return True\n\n    @classmethod\n    def _from_arff_file(cls, filename: Path) -&gt; OpenMLSplit:  # noqa: C901, PLR0912\n        repetitions = None\n        name = None\n\n        pkl_filename = filename.with_suffix(\".pkl.py3\")\n\n        if pkl_filename.exists():\n            with pkl_filename.open(\"rb\") as fh:\n                # TODO(eddiebergman): Would be good to figure out what _split is and assert it is\n                _split = pickle.load(fh)  # noqa: S301\n            repetitions = _split[\"repetitions\"]\n            name = _split[\"name\"]\n\n        # Cache miss\n        if repetitions is None:\n            # Faster than liac-arff and sufficient in this situation!\n            if not filename.exists():\n                raise FileNotFoundError(f\"Split arff {filename} does not exist!\")\n\n            file_data = arff.load(filename.open(\"r\"), return_type=arff.DENSE_GEN)\n            splits = file_data[\"data\"]\n            name = file_data[\"relation\"]\n            attrnames = [attr[0] for attr in file_data[\"attributes\"]]\n\n            repetitions = OrderedDict()\n\n            type_idx = attrnames.index(\"type\")\n            rowid_idx = attrnames.index(\"rowid\")\n            repeat_idx = attrnames.index(\"repeat\")\n            fold_idx = attrnames.index(\"fold\")\n            sample_idx = attrnames.index(\"sample\") if \"sample\" in attrnames else None\n\n            for line in splits:\n                # A line looks like type, rowid, repeat, fold\n                repetition = int(line[repeat_idx])\n                fold = int(line[fold_idx])\n                sample = 0\n                if sample_idx is not None:\n                    sample = int(line[sample_idx])\n\n                if repetition not in repetitions:\n                    repetitions[repetition] = OrderedDict()\n                if fold not in repetitions[repetition]:\n                    repetitions[repetition][fold] = OrderedDict()\n                if sample not in repetitions[repetition][fold]:\n                    repetitions[repetition][fold][sample] = ([], [])\n                split = repetitions[repetition][fold][sample]\n\n                type_ = line[type_idx]\n                if type_ == \"TRAIN\":\n                    split[0].append(line[rowid_idx])\n                elif type_ == \"TEST\":\n                    split[1].append(line[rowid_idx])\n                else:\n                    raise ValueError(type_)\n\n            for repetition in repetitions:\n                for fold in repetitions[repetition]:\n                    for sample in repetitions[repetition][fold]:\n                        repetitions[repetition][fold][sample] = Split(\n                            np.array(repetitions[repetition][fold][sample][0], dtype=np.int32),\n                            np.array(repetitions[repetition][fold][sample][1], dtype=np.int32),\n                        )\n\n            with pkl_filename.open(\"wb\") as fh:\n                pickle.dump({\"name\": name, \"repetitions\": repetitions}, fh, protocol=2)\n\n        assert name is not None\n        return cls(name, \"\", repetitions)\n\n    def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n        Parameters\n        ----------\n        repeat : int\n            Index of the repeat to retrieve.\n        fold : int\n            Index of the fold to retrieve.\n        sample : int\n            Index of the sample to retrieve.\n\n        Returns\n        -------\n        numpy.ndarray\n            The data split for the specified repeat, fold, and sample.\n\n        Raises\n        ------\n        ValueError\n            If the specified repeat, fold, or sample is not known.\n        \"\"\"\n        if repeat not in self.split:\n            raise ValueError(\"Repeat %s not known\" % str(repeat))\n        if fold not in self.split[repeat]:\n            raise ValueError(\"Fold %s not known\" % str(fold))\n        if sample not in self.split[repeat][fold]:\n            raise ValueError(\"Sample %s not known\" % str(sample))\n        return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/split/#openml.tasks.split.OpenMLSplit.get","title":"<code>get(repeat=0, fold=0, sample=0)</code>","text":"<p>Returns the specified data split from the CrossValidationSplit object.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>Index of the repeat to retrieve.</p> <code>0</code> <code>fold</code> <code>int</code> <p>Index of the fold to retrieve.</p> <code>0</code> <code>sample</code> <code>int</code> <p>Index of the sample to retrieve.</p> <code>0</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>The data split for the specified repeat, fold, and sample.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the specified repeat, fold, or sample is not known.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n    Parameters\n    ----------\n    repeat : int\n        Index of the repeat to retrieve.\n    fold : int\n        Index of the fold to retrieve.\n    sample : int\n        Index of the sample to retrieve.\n\n    Returns\n    -------\n    numpy.ndarray\n        The data split for the specified repeat, fold, and sample.\n\n    Raises\n    ------\n    ValueError\n        If the specified repeat, fold, or sample is not known.\n    \"\"\"\n    if repeat not in self.split:\n        raise ValueError(\"Repeat %s not known\" % str(repeat))\n    if fold not in self.split[repeat]:\n        raise ValueError(\"Fold %s not known\" % str(fold))\n    if sample not in self.split[repeat][fold]:\n        raise ValueError(\"Sample %s not known\" % str(sample))\n    return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/split/#openml.tasks.split.Split","title":"<code>Split</code>","text":"<p>               Bases: <code>NamedTuple</code></p> <p>A single split of a dataset.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>class Split(NamedTuple):\n    \"\"\"A single split of a dataset.\"\"\"\n\n    train: np.ndarray\n    test: np.ndarray\n</code></pre>"},{"location":"reference/tasks/task/","title":"task","text":""},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLClassificationTask","title":"<code>OpenMLClassificationTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Classification task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the Classification task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the Classification task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target variable.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the Classification task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the Classification task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the Classification task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Classification task (if it already exists on OpenML).</p> <code>None</code> <code>class_labels</code> <code>List of str</code> <p>A list of class labels (for classification tasks).</p> <code>None</code> <code>cost_matrix</code> <code>array</code> <p>A cost matrix (for classification tasks).</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClassificationTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Classification task type.\n    task_type : str\n        Name of the Classification task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the Classification task.\n    target_name : str\n        Name of the target variable.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the Classification task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the Classification task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure.\n    data_splits_url : str, default=None\n        URL of the data splits for the Classification task.\n    task_id : Union[int, None]\n        ID of the Classification task (if it already exists on OpenML).\n    class_labels : List of str, default=None\n        A list of class labels (for classification tasks).\n    cost_matrix : array, default=None\n        A cost matrix (for classification tasks).\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n        self.class_labels = class_labels\n        self.cost_matrix = cost_matrix\n\n        if cost_matrix is not None:\n            raise NotImplementedError(\"Costmatrix\")\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLClusteringTask","title":"<code>OpenMLClusteringTask</code>","text":"<p>               Bases: <code>OpenMLTask</code></p> <p>OpenML Clustering object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML clustering task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML clustering task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset used in clustering the task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML clustering task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure used in the clustering task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the clustering task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the clustering task.</p> <code>None</code> <code>target_name</code> <code>str</code> <p>Name of the target feature (class) that is not part of the feature set for the clustering task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClusteringTask(OpenMLTask):\n    \"\"\"OpenML Clustering object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML clustering task.\n    task_type : str\n        Task type of the OpenML clustering task.\n    data_set_id : int\n        ID of the OpenML dataset used in clustering the task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    task_id : Union[int, None]\n        ID of the OpenML clustering task.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure used in the clustering task.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the clustering task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the clustering task.\n    target_name : str, default=None\n        Name of the target feature (class) that is not part of the\n        feature set for the clustering task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 17,\n        task_id: int | None = None,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        evaluation_measure: str | None = None,\n        target_name: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            evaluation_measure=evaluation_measure,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\"] = \"array\",\n    ) -&gt; np.ndarray | scipy.sparse.spmatrix:\n        ...\n\n    @overload\n    def get_X(self, dataset_format: Literal[\"dataframe\"]) -&gt; pd.DataFrame:\n        ...\n\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n    ) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        dataset = self.get_dataset()\n        data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n        return data\n\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        # Right now, it is not supported as a feature.\n        # Uncomment if it is supported on the server\n        # in the future.\n        # https://github.com/openml/OpenML/issues/925\n        \"\"\"\n        task_dict = task_container['oml:task_inputs']\n        if self.target_name is not None:\n            task_dict['oml:input'].append(\n                OrderedDict([\n                    ('@name', 'target_feature'),\n                    ('#text', self.target_name)\n                ])\n            )\n        \"\"\"\n        return super()._to_dict()\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLClusteringTask.get_X","title":"<code>get_X(dataset_format='array')</code>","text":"<pre><code>get_X(dataset_format: Literal['array'] = 'array') -&gt; np.ndarray | scipy.sparse.spmatrix\n</code></pre><pre><code>get_X(dataset_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X(\n    self,\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    dataset = self.get_dataset()\n    data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n    return data\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLLearningCurveTask","title":"<code>OpenMLLearningCurveTask</code>","text":"<p>               Bases: <code>OpenMLClassificationTask</code></p> <p>OpenML Learning Curve object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Learning Curve task.</p> required <code>task_type</code> <code>str</code> <p>Name of the Learning Curve task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the dataset that this task is associated with.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature in the dataset.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure to use for evaluating models.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Additional parameters for the estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the file containing the data splits for Learning Curve task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Learning Curve task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure to use for evaluating models.</p> <code>None</code> <code>class_labels</code> <code>list of str</code> <p>Class labels for Learning Curve tasks.</p> <code>None</code> <code>cost_matrix</code> <code>numpy array</code> <p>Cost matrix for Learning Curve tasks.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLLearningCurveTask(OpenMLClassificationTask):\n    \"\"\"OpenML Learning Curve object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Learning Curve task.\n    task_type : str\n        Name of the Learning Curve task.\n    data_set_id : int\n        ID of the dataset that this task is associated with.\n    target_name : str\n        Name of the target feature in the dataset.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure to use for evaluating models.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Additional parameters for the estimation procedure.\n    data_splits_url : str, default=None\n        URL of the file containing the data splits for Learning Curve task.\n    task_id : Union[int, None]\n        ID of the Learning Curve task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure to use for evaluating models.\n    class_labels : list of str, default=None\n        Class labels for Learning Curve tasks.\n    cost_matrix : numpy array, default=None\n        Cost matrix for Learning Curve tasks.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 13,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n            class_labels=class_labels,\n            cost_matrix=cost_matrix,\n        )\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLRegressionTask","title":"<code>OpenMLRegressionTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Regression object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML Regression task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML Regression task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature used in the Regression task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the Regression task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML Regression task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the Regression task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLRegressionTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Regression object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML Regression task.\n    task_type : str\n        Task type of the OpenML Regression task.\n    data_set_id : int\n        ID of the OpenML dataset.\n    target_name : str\n        Name of the target feature used in the Regression task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the Regression task.\n    task_id : Union[int, None]\n        ID of the OpenML Regression task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the Regression task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 7,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLSupervisedTask","title":"<code>OpenMLSupervisedTask</code>","text":"<p>               Bases: <code>OpenMLTask</code>, <code>ABC</code></p> <p>OpenML Supervised Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature (the class variable).</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure for the task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure for the task.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the task.</p> <code>None</code> <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLSupervisedTask(OpenMLTask, ABC):\n    \"\"\"OpenML Supervised Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the task type.\n    task_type : str\n        Name of the task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the task.\n    target_name : str\n        Name of the target feature (the class variable).\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure for the task.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure for the task.\n    data_splits_url : str, default=None\n        URL of the data splits for the task.\n    task_id: Union[int, None]\n        Refers to the unique identifier of task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | scipy.sparse.spmatrix,\n        np.ndarray | None,\n    ]:\n        ...\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\"]\n    ) -&gt; tuple[\n        pd.DataFrame,\n        pd.Series | pd.DataFrame | None,\n    ]:\n        ...\n\n    # TODO(eddiebergman): Do all OpenMLSupervisedTask have a `y`?\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n        np.ndarray | pd.Series | pd.DataFrame | None,\n    ]:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        dataset = self.get_dataset()\n        if self.task_type_id not in (\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.SUPERVISED_REGRESSION,\n            TaskType.LEARNING_CURVE,\n        ):\n            raise NotImplementedError(self.task_type)\n\n        X, y, _, _ = dataset.get_data(\n            dataset_format=dataset_format,\n            target=self.target_name,\n        )\n        return X, y\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        task_container = super()._to_dict()\n        oml_input = task_container[\"oml:task_inputs\"][\"oml:input\"]  # type: ignore\n        assert isinstance(oml_input, list)\n\n        oml_input.append({\"@name\": \"target_feature\", \"#text\": self.target_name})\n        return task_container\n\n    @property\n    def estimation_parameters(self) -&gt; dict[str, str] | None:\n        \"\"\"Return the estimation parameters for the task.\"\"\"\n        warnings.warn(\n            \"The estimation_parameters attribute will be \"\n            \"deprecated in the future, please use \"\n            \"estimation_procedure['parameters'] instead\",\n            PendingDeprecationWarning,\n            stacklevel=2,\n        )\n        return self.estimation_procedure[\"parameters\"]\n\n    @estimation_parameters.setter\n    def estimation_parameters(self, est_parameters: dict[str, str] | None) -&gt; None:\n        self.estimation_procedure[\"parameters\"] = est_parameters\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLSupervisedTask.estimation_parameters","title":"<code>estimation_parameters: dict[str, str] | None</code>  <code>property</code> <code>writable</code>","text":"<p>Return the estimation parameters for the task.</p>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLSupervisedTask.get_X_and_y","title":"<code>get_X_and_y(dataset_format='array')</code>","text":"<pre><code>get_X_and_y(dataset_format: Literal['array'] = 'array') -&gt; tuple[np.ndarray | scipy.sparse.spmatrix, np.ndarray | None]\n</code></pre><pre><code>get_X_and_y(dataset_format: Literal['dataframe']) -&gt; tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X_and_y(\n    self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n    np.ndarray | pd.Series | pd.DataFrame | None,\n]:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    dataset = self.get_dataset()\n    if self.task_type_id not in (\n        TaskType.SUPERVISED_CLASSIFICATION,\n        TaskType.SUPERVISED_REGRESSION,\n        TaskType.LEARNING_CURVE,\n    ):\n        raise NotImplementedError(self.task_type)\n\n    X, y, _, _ = dataset.get_data(\n        dataset_format=dataset_format,\n        target=self.target_name,\n    )\n    return X, y\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask","title":"<code>OpenMLTask</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Task object.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of OpenML task.</p> required <code>task_type_id</code> <code>TaskType</code> <p>Refers to the type of OpenML task.</p> required <code>task_type</code> <code>str</code> <p>Refers to the OpenML task.</p> required <code>data_set_id</code> <code>int</code> <p>Refers to the data.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>Refers to the type of estimates used.</p> <code>1</code> <code>estimation_procedure_type</code> <code>str | None</code> <p>Refers to the type of estimation procedure used for the OpenML task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict[str, str] | None</code> <p>Estimation parameters used for the OpenML task.</p> <code>None</code> <code>evaluation_measure</code> <code>str | None</code> <p>Refers to the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str | None</code> <p>Refers to the URL of the data splits used for the OpenML task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLTask(OpenMLBase):\n    \"\"\"OpenML Task object.\n\n    Parameters\n    ----------\n    task_id: Union[int, None]\n        Refers to the unique identifier of OpenML task.\n    task_type_id: TaskType\n        Refers to the type of OpenML task.\n    task_type: str\n        Refers to the OpenML task.\n    data_set_id: int\n        Refers to the data.\n    estimation_procedure_id: int\n        Refers to the type of estimates used.\n    estimation_procedure_type: str, default=None\n        Refers to the type of estimation procedure used for the OpenML task.\n    estimation_parameters: [Dict[str, str]], default=None\n        Estimation parameters used for the OpenML task.\n    evaluation_measure: str, default=None\n        Refers to the evaluation measure.\n    data_splits_url: str, default=None\n        Refers to the URL of the data splits used for the OpenML task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int | None,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n    ):\n        self.task_id = int(task_id) if task_id is not None else None\n        self.task_type_id = task_type_id\n        self.task_type = task_type\n        self.dataset_id = int(data_set_id)\n        self.evaluation_measure = evaluation_measure\n        self.estimation_procedure: _EstimationProcedure = {\n            \"type\": estimation_procedure_type,\n            \"parameters\": estimation_parameters,\n            \"data_splits_url\": data_splits_url,\n        }\n        self.estimation_procedure_id = estimation_procedure_id\n        self.split: OpenMLSplit | None = None\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"t\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the OpenML ID of this task.\"\"\"\n        return self.task_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        base_server_url = openml.config.get_server_base_url()\n        fields: dict[str, Any] = {\n            \"Task Type Description\": f\"{base_server_url}/tt/{self.task_type_id}\"\n        }\n        if self.task_id is not None:\n            fields[\"Task ID\"] = self.task_id\n            fields[\"Task URL\"] = self.openml_url\n        if self.evaluation_measure is not None:\n            fields[\"Evaluation Measure\"] = self.evaluation_measure\n        if self.estimation_procedure is not None:\n            fields[\"Estimation Procedure\"] = self.estimation_procedure[\"type\"]\n\n        # TODO(eddiebergman): Subclasses could advertise/provide this, instead of having to\n        # have the base class know about it's subclasses.\n        target_name = getattr(self, \"target_name\", None)\n        if target_name is not None:\n            fields[\"Target Feature\"] = target_name\n\n            class_labels = getattr(self, \"class_labels\", None)\n            if class_labels is not None:\n                fields[\"# of Classes\"] = len(class_labels)\n\n            if hasattr(self, \"cost_matrix\"):\n                fields[\"Cost Matrix\"] = \"Available\"\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Task Type Description\",\n            \"Task ID\",\n            \"Task URL\",\n            \"Estimation Procedure\",\n            \"Evaluation Measure\",\n            \"Target Feature\",\n            \"# of Classes\",\n            \"Cost Matrix\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def get_dataset(self) -&gt; datasets.OpenMLDataset:\n        \"\"\"Download dataset associated with task.\"\"\"\n        return datasets.get_dataset(self.dataset_id)\n\n    def get_train_test_split_indices(\n        self,\n        fold: int = 0,\n        repeat: int = 0,\n        sample: int = 0,\n    ) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n        # Replace with retrieve from cache\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.get(repeat=repeat, fold=fold, sample=sample)\n\n    def _download_split(self, cache_file: Path) -&gt; None:\n        # TODO(eddiebergman): Not sure about this try to read and error approach\n        try:\n            with cache_file.open(encoding=\"utf8\"):\n                pass\n        except OSError:\n            split_url = self.estimation_procedure[\"data_splits_url\"]\n            openml._api_calls._download_text_file(\n                source=str(split_url),\n                output_path=str(cache_file),\n            )\n\n    def download_split(self) -&gt; OpenMLSplit:\n        \"\"\"Download the OpenML split for a given task.\"\"\"\n        # TODO(eddiebergman): Can this every be `None`?\n        assert self.task_id is not None\n        cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n        cached_split_file = cache_dir / \"datasplits.arff\"\n\n        try:\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n        except OSError:\n            # Next, download and cache the associated split file\n            self._download_split(cached_split_file)\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n\n        return split\n\n    def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n        \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.repeats, self.split.folds, self.split.samples\n\n    # TODO(eddiebergman): Really need some better typing on all this\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        \"\"\"Creates a dictionary representation of self in a string format (for XML parsing).\"\"\"\n        oml_input = [\n            {\"@name\": \"source_data\", \"#text\": str(self.dataset_id)},\n            {\"@name\": \"estimation_procedure\", \"#text\": str(self.estimation_procedure_id)},\n        ]\n        if self.evaluation_measure is not None:  #\n            oml_input.append({\"@name\": \"evaluation_measures\", \"#text\": self.evaluation_measure})\n\n        return {\n            \"oml:task_inputs\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                \"oml:task_type_id\": self.task_type_id.value,  # This is an int from the enum?\n                \"oml:input\": oml_input,\n            }\n        }\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.task_id = int(xml_response[\"oml:upload_task\"][\"oml:id\"])\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the OpenML ID of this task.</p>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.download_split","title":"<code>download_split()</code>","text":"<p>Download the OpenML split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def download_split(self) -&gt; OpenMLSplit:\n    \"\"\"Download the OpenML split for a given task.\"\"\"\n    # TODO(eddiebergman): Can this every be `None`?\n    assert self.task_id is not None\n    cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n    cached_split_file = cache_dir / \"datasplits.arff\"\n\n    try:\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n    except OSError:\n        # Next, download and cache the associated split file\n        self._download_split(cached_split_file)\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n\n    return split\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.get_dataset","title":"<code>get_dataset()</code>","text":"<p>Download dataset associated with task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_dataset(self) -&gt; datasets.OpenMLDataset:\n    \"\"\"Download dataset associated with task.\"\"\"\n    return datasets.get_dataset(self.dataset_id)\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.get_split_dimensions","title":"<code>get_split_dimensions()</code>","text":"<p>Get the (repeats, folds, samples) of the split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n    \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.repeats, self.split.folds, self.split.samples\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.get_train_test_split_indices","title":"<code>get_train_test_split_indices(fold=0, repeat=0, sample=0)</code>","text":"<p>Get the indices of the train and test splits for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_train_test_split_indices(\n    self,\n    fold: int = 0,\n    repeat: int = 0,\n    sample: int = 0,\n) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n    # Replace with retrieve from cache\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.get(repeat=repeat, fold=fold, sample=sample)\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.TaskType","title":"<code>TaskType</code>","text":"<p>               Bases: <code>Enum</code></p> <p>Possible task types as defined in OpenML.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>class TaskType(Enum):\n    \"\"\"Possible task types as defined in OpenML.\"\"\"\n\n    SUPERVISED_CLASSIFICATION = 1\n    SUPERVISED_REGRESSION = 2\n    LEARNING_CURVE = 3\n    SUPERVISED_DATASTREAM_CLASSIFICATION = 4\n    CLUSTERING = 5\n    MACHINE_LEARNING_CHALLENGE = 6\n    SURVIVAL_ANALYSIS = 7\n    SUBGROUP_DISCOVERY = 8\n    MULTITASK_REGRESSION = 9\n</code></pre>"},{"location":"services/","title":"services","text":"<p>Overview of all OpenML components including a docker-compose to run OpenML services locally</p>"},{"location":"services/#overview","title":"Overview","text":""},{"location":"services/#prerequisites","title":"Prerequisites","text":"<ul> <li>Linux/MacOS/Windows (should all work)</li> <li>Docker </li> <li>Docker Compose version 2.21.0 or higher</li> </ul>"},{"location":"services/#usage","title":"Usage","text":"<p>You run all OpenML services locally using <pre><code>docker compose --profile all up -d\n</code></pre> Stop it again using  <pre><code>docker compose --profile all down\n</code></pre></p>"},{"location":"services/#profiles","title":"Profiles","text":"<p>You can use different profiles:</p> <ul> <li><code>[no profile]</code>: databases</li> <li><code>\"elasticsearch\"</code>: databases + elasticsearch</li> <li><code>\"rest-api\"</code>: databases + elasticsearch + REST API</li> <li><code>\"frontend\"</code>: databases + elasticsearch + REST API + frontend + email-server</li> <li><code>\"minio\"</code>: database + elasticsearch + REST APP + MinIO + parquet and croissant conversion</li> <li><code>\"all\"</code>: everything</li> </ul> <p>Usage examples: <pre><code>docker compose --profile all up -d       # all services\ndocker compose up -d                     # only the database\ndocker compose --profile frontend up -d  # Frontend, rest-api, elasticsearch and database\n</code></pre> Use the same profile for your <code>down</code> command.</p>"},{"location":"services/#known-issues","title":"Known issues","text":"<p>See the Github Issue list for the known issues.</p>"},{"location":"services/#debugging","title":"Debugging","text":"<p>Some usefull commands: <pre><code>docker logs openml-php-rest-api -f              # tail the logs of the php rest api\ndocker exec -it openml-php-rest-api /bin/bash   # go into the php rest api container\n./scripts/connect_db.sql                        # access the database\n</code></pre></p>"},{"location":"services/#endpoints","title":"Endpoints","text":"<p>[!TIP] If you change any port, make sure to change it for all services! The elasticsearch config, for instance, needs to know the port of the frontend (for CORS).</p> <p>When you spin up the docker-compose, you'll get these endpoints: - Frontend: localhost:5000 - Database: localhost:3306, filled with test data. - ElasticSearch: localhost:9200, filled with test data. - Rest API: localhost:8080 - Minio: console at localhost:9001, filled with test data.</p>"},{"location":"services/#credentials","title":"Credentials","text":"<p>The credentials for the database can be found in <code>config/database/.env</code>, for minio in <code>config/minio/.env</code>, etc.</p>"},{"location":"services/#emails","title":"Emails","text":"<p>The email-server is used for emails from the frontend. For example, if you create a new user, an  email is send to the user. All outgoing emails are rerouted to catchall@example.com. You can see  the messages in <code>config/email-server/messages</code>. Note that some of the urls in the emails need to  be slightly altered to use them in the test setup: change https to http.</p>"},{"location":"services/#development","title":"Development","text":""},{"location":"services/#php-parquet-and-croissant-converter","title":"PHP, Parquet and Croissant converter","text":"<p>If you want to do local development on containers that are part of the docker-compose, you want those containers to change based on your code. You should have the relevant code somewhere on your system, you only need to tell the docker-compose where to find it. You can do so by setting environment variables. </p> <p>Create a <code>.env</code> file inside this directory, and set:</p>"},{"location":"services/#php","title":"PHP","text":"<pre><code>PHP_CODE_DIR=/path/to/OpenML                  # Root of https://github.com/openml/OpenML on your computer\nPHP_CODE_VAR_WWW_OPENML=/var/www/openml       # Always set this to /var/www/openml. Leave empty if you leave PHP_CODE_DIR empty\n</code></pre> <p>Make sure to create <code>openml_OS/config/BASE_CONFIG.php</code> in your local <code>$PHP_CODE_DIR</code>. The correct configuration can be found in <code>config/php.env</code>. Run docker compose with profile <code>rest-api</code>.</p>"},{"location":"services/#parquet","title":"Parquet","text":"<pre><code>ARFF_TO_PQ_CODE_DIR=/path/to/minio-data       # Root of https://github.com/openml-labs/minio-data on your computer\nARFF_TO_PQ_APP=/app                           # Always set this to /app. Leave empty if you leave ARFF_TO_PQ_CODE_DIR empty\n</code></pre>"},{"location":"services/#croissant","title":"Croissant","text":"<pre><code>CROISSANT_CODE_DIR=/path/to/openml-croissant/python  # Python directory of https://github.com/openml/openml-croissant on your computer\nCROISSANT_APP=/app                                   # Always set this to /app. Leave empty if you leave CROISSANT_CODE_DIR empty\n</code></pre>"},{"location":"services/#frontend","title":"Frontend","text":"<pre><code>FRONTEND_CODE_DIR=/path/to/openml.org        # Python directory of https://github.com/openml/openml.org on your computer\nFRONTEND_APP=/app                            # Always set this to /app. Leave empty if you leave FRONTEND_CODE_DIR empty\n</code></pre>"},{"location":"services/#other-services","title":"Other services","text":"<p>If you want to develop a service that depends on any of the services in this docker-compose, just bring up this docker-compose and point your service to the correct endpoints.</p>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-\\.]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Introduction","text":"An open, automated, and frictionless machine learning environment. <p> \u00a0 1000s of data sets, uniformly formatted, easy to load, organized online</p> <p> \u00a0Models and pipelines automatically uploaded from machine learning libraries</p> <p> Extensive APIs to integrate OpenML into your tools and scripts</p> <p>  Easily reproducible results (e.g. models, evaluations) for comparison and reuse</p> <p>\u00a0 Stand on the shoulders of giants, and collaborate in real time</p> <p>\u00a0 Make your work more visible and reusable</p> <p>\u00a0 Built for automation: streamline your experiments and model building</p>"},{"location":"#installation","title":"Installation","text":"<p>The OpenML package is available in many languages and across libraries. For more information about them, see the Integrations page.</p> Python/sklearnPytorchKerasTensorFlowRJuliaRUST.Net <ul> <li>Python/sklearn repository</li> <li><code>pip install openml</code></li> </ul> <ul> <li>Pytorch repository</li> <li><code>pip install openml-pytorch</code></li> </ul> <ul> <li>Keras repository</li> <li><code>pip install openml-keras</code></li> </ul> <ul> <li>TensorFlow repository</li> <li><code>pip install openml-tensorflow</code></li> </ul> <ul> <li>R repository</li> <li><code>install.packages(\"mlr3oml\")</code></li> </ul> <ul> <li>Julia repository</li> <li><code>using Pkg;Pkg.add(\"OpenML\")</code></li> </ul> <ul> <li>RUST repository</li> <li>Install from source</li> </ul> <ul> <li>.Net repository</li> <li><code>Install-Package openMl</code></li> </ul> <p>You might also need to set up the API key. For more information, see Authentication.</p>"},{"location":"#learning-openml","title":"Learning OpenML","text":"<p>Aside from the individual package documentations, you can learn more about OpenML through the following resources: The core concepts of OpenML are explained in the Concepts page. These concepts include the principle behind using Datasets, Runs, Tasks, Flows, Benchmarking and much more. Going through them will help you leverage OpenML even better in your work.</p>"},{"location":"#contributing-to-openml","title":"Contributing to OpenML","text":"<p>OpenML is an open source project, hosted on GitHub. We welcome everybody to help improve OpenML, and make it more useful for everyone. For more information on how to contribute, see the Contributing page.</p> <p>We want to make machine learning and data analysis simple, accessible, collaborative and open with an optimal division of labour between computers and humans.</p>"},{"location":"#want-to-get-involved","title":"Want to get involved?","text":"<p>Awesome, we're happy to have you! </p> <p>OpenML is dependent on the community. If you want to help, please email us (openmlHQ@googlegroups.com). If you feel already comfortable you can help by opening issues or make a pull request on GitHub. We also have regular workshops you can join (they are announced on openml.org).</p>"},{"location":"apiv2/","title":"OpenML Server API Software Documentation","text":"<p>This is the Python-based OpenML REST API server. It's a rewrite of our old backend built with a modern Python-based stack.</p> <p>Looking to access data on OpenML?</p> <p>If you simply want to access data stored on OpenML in a programmatic way, please have a look at connector packages in Python, Java, or R.</p> <p>If you are looking to interface directly with the REST API, and are looking for documentation on the REST API endpoints, visit the APIs page.</p> <p>This documentation is for developing and hosting your own OpenML REST API.</p>"},{"location":"apiv2/#development-roadmap","title":"Development Roadmap","text":"<p>First we will mimic current server functionality, relying on many implementation details present in the current production server. We will implement all endpoints using the SQL text queries based on PHP implementation, which should give near-identical responses to the current JSON endpoints. Minor exceptions are permitted but will be documented.</p> <p>At the same time we may also provide a work-in-progress \"new\" endpoint, but there won't be official support for it at this stage. After we verify the output of the endpoints are identical (minus any intentional documented differences), we will officially release the new API. The old API will remain available. After that, we can start working on a new version of the JSON API which is more standardized, leverages typing, and so on:</p> <ul> <li>Clean up the database: standardize value formats where possible (e.g., (un)quoting    contributor names in the dataset's contributor field), and add database level    constraints on new values.</li> <li>Redesign what the new API responses should look like and implement them,    API will be available to the public as it is developed.</li> <li>Refactor code-base to use ORM (using <code>SQLAlchemy</code>, <code>SQLModel</code>, or similar).</li> <li>Officially release the modernized API.</li> </ul> <p>There is no planned sunset date for the old API. This will depend on the progress with the new API as well as the usage numbers of the old API.</p>"},{"location":"apiv2/installation/","title":"Installation","text":"<p>Current instructions tested on Mac, but likely work on most Unix systems.</p> <p>The OpenML server will be developed and maintained for the latest minor release of Python (Python 3.12 as of writing). You can install the dependencies locally or work with docker containers.</p> Use <code>pyenv</code> to manage Python installations <p>We recommend using <code>pyenv</code> if you are working with multiple local Python versions. After following the installation instructions for <code>pyenv</code> check that you can execute it:</p> <pre><code>&gt; pyenv local\n3.12\n</code></pre> <p>If <code>pyenv</code> can't be found, please make sure to update the terminal environment (either by <code>reset</code>ing it, or by closing and opening the terminal). If you get the message <code>pyenv: no local version configured for this directory</code> first clone the repository as described below and try again from the root of the cloned repository.</p> <p>You can then install the Python version this project uses with: <code>cat .python-version | pyenv install</code></p>"},{"location":"apiv2/installation/#local-installation","title":"Local Installation","text":"<p>These instructions assume Python 3.12 and git are already installed.</p> <p>You may need to install Python3 and MySQL development headers.</p> <p>It may be necessary to first install additional headers before proceeding with a local installation of the <code>mysqlclient</code> dependency. They are documented under \"Installation\" of the <code>mysqlclient</code> documentation.</p> For UsersFor Contributors <p>If you don't plan to make code changes, you can install directly from Github. We recommend to install the OpenML server and its dependencies into a new virtual environment. Installing the project into a new virtual environment<pre><code>python -m venv venv\nsource venv/bin/activate\n\npython -m pip install git+https://github.com/openml/server-api.git\n</code></pre> If you do plan to make code changes, we recommend you follow the instructions under the \"For Contributors\" tab, even if you do not plan to contribute your changes back into the project.</p> <p>If you plan to make changes to this project, it will be useful to install the project from a cloned fork. To fork the project, go to our project page and click \"fork\". This makes a copy of the repository under your own Github account. You can then clone your own fork (replace <code>USER_NAME</code> with your Github username):</p> Cloning your fork<pre><code>git clone https://github.com/USER_NAME/server-api.git\ncd server-api\n</code></pre> <p>Then we can install the project into a new virtual environment in edit mode:</p> <p>Installing the project into a new virtual environment<pre><code>python -m venv venv\nsource venv/bin/activate\n\npython -m pip install -e \".[dev,docs]\"\n</code></pre> Note that this also installs optional dependencies for development and documentation tools. We require this for contributors, but we also highly recommend it anyone that plans to make code changes.</p>"},{"location":"apiv2/installation/#setting-up-a-database-server","title":"Setting up a Database Server","text":"<p>Depending on your use of the server, there are multiple ways to set up your own OpenML database. To simply connect to an existing database, see configuring the REST API Server below.</p>"},{"location":"apiv2/installation/#setting-up-a-new-database","title":"Setting up a new database","text":"<p>This sets up an entirely empty database with the expected OpenML tables in place. This is intended for new deployments of OpenML, for example to host a private OpenML server.</p> <p>Instructions are incomplete. See issue#78.</p>"},{"location":"apiv2/installation/#setting-up-a-test-database","title":"Setting up a test database","text":"<p>We provide a prebuilt docker image that already contains test data.</p> Docker ComposeDocker Run <p>To start the database through <code>docker compose</code>, run:</p> <pre><code>docker compose up database\n</code></pre> <p>which starts a database.</p> <p>To start a test database as stand-alone container, run:</p> <pre><code>docker run  --rm -e MYSQL_ROOT_PASSWORD=ok -p 3306:3306 -d --name openml-test-database openml/test-database:latest\n</code></pre> <p>You may opt to add the container to a network instead, to make it reachable from other docker containers:</p> <pre><code>docker network create openml\ndocker run  --rm -e MYSQL_ROOT_PASSWORD=ok -p 3306:3306 -d --name openml-test-database --network openml openml/test-database:latest\n</code></pre> <p>The container may take a minute to initialise, but afterwards you can connect to it. Either from a local <code>mysql</code> client at <code>127.0.0.1:3306</code> or from a docker container on the same network. For example:</p> <p><pre><code>docker run --network NETWORK --rm -it mysql mysql -hopenml-test-database -uroot -pok\n</code></pre> where <code>NETWORK</code> is <code>openml</code> when using <code>docker run</code> when following the example, and <code>NETWORK</code> is <code>server-api_default</code> if you used <code>docker compose</code> (specifically, it is <code>DIRECTORY_NAME</code> + <code>_default</code>, so if you renamed the <code>server-api</code> directory to something else, the network name reflects that).</p>"},{"location":"apiv2/installation/#configuring-the-rest-api-server","title":"Configuring the REST API Server","text":"<p>The REST API is configured through a TOML file.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/migration/","title":"Migration","text":"<p>The Python reimplementation provides the same endpoints as the old API, which are largely functioning the same way. However, there are a few major deviations:</p> <ul> <li>Use of typed JSON: e.g., when a value represents an integer, it is returned as integer.</li> <li>Lists when multiple values are possible: if a field can have none, one, or multiple entries (e.g., authors), we always return a list.</li> <li>Restriction or expansion of input types as appropriate.</li> <li>Standardizing authentication and access messages, and consistently execute those checks   before fetching data or providing error messages about the data.</li> </ul> <p>The list above is not exhaustive. Minor changes include, for example, bug fixes and the removal of unnecessary nesting. There may be undocumented changes, especially in edge cases which may not have occurred in the test environment. As the PHP API was underspecified, the re-implementation is based on a mix of reading old code and probing the API. If there is a behavioral change which was not documented but affects you, please open a bug report.</p>"},{"location":"apiv2/migration/#all-endpoints","title":"All Endpoints","text":"<p>The following changes affect all endpoints.</p>"},{"location":"apiv2/migration/#error-on-invalid-input","title":"Error on Invalid Input","text":"<p>When providing input of invalid types (e.g., a non-integer dataset id) the HTTP header and JSON content will be different.</p> HTTP Header<pre><code>- 412 Precondition Failed\n+ 422 Unprocessable Entity\n</code></pre> JSON Content<pre><code>- {\"error\":{\"code\":\"100\",\"message\":\"Function not valid\"}}\n+ {\"detail\":[{\"loc\":[\"query\",\"_dataset_id\"],\"msg\":\"value is not a valid integer\",\"type\":\"type_error.integer\"}]}\n</code></pre> <p>Input validation has been added to many end points</p> <p>There are endpoints which previously did not do any input validation.    These endpoints now do enforce stricter input constraints.    Constraints for each endpoint parameter are documented in the API docs.</p>"},{"location":"apiv2/migration/#other-errors","title":"Other Errors","text":"<p>For any other error messages, the response is identical except that outer field will be <code>\"detail\"</code> instead of <code>\"error\"</code>:</p> JSON Content<pre><code>- {\"error\":{\"code\":\"112\",\"message\":\"No access granted\"}}\n+ {\"detail\":{\"code\":\"112\",\"message\":\"No access granted\"}}\n</code></pre> <p>In some cases the JSON endpoints previously returned XML (example), the new API always returns JSON.</p> XML replaced by JSON<pre><code>- &lt;oml:error xmlns:oml=\"http://openml.org/openml\"&gt;\n-   &lt;oml:code&gt;103&lt;/oml:code&gt;\n-   &lt;oml:message&gt;Authentication failed&lt;/oml:message&gt;\n- &lt;/oml:error&gt;\n+ {\"detail\": {\"code\":\"103\", \"message\": \"Authentication failed\" } }\n</code></pre>"},{"location":"apiv2/migration/#datasets","title":"Datasets","text":""},{"location":"apiv2/migration/#get-dataset_id","title":"<code>GET /{dataset_id}</code>","text":"<ul> <li>Dataset format names are normalized to be all lower-case    (<code>\"Sparse_ARFF\"</code> -&gt;  <code>\"sparse_arff\"</code>).</li> <li>Non-<code>arff</code> datasets will not incorrectly have a <code>\"parquet_url\"</code> (openml#1189).</li> <li>If <code>\"creator\"</code> contains multiple comma-separated creators it is always returned    as a list, instead of it depending on the quotation used by the original uploader.</li> <li>For (some?) datasets that have multiple values in <code>\"ignore_attribute\"</code>, this field    is correctly populated instead of omitted.</li> <li>Processing date is formatted with a <code>T</code> in the middle:   processing_date<pre><code>- \"2019-07-09 15:22:03\"\n+ \"2019-07-09T15:22:03\"\n</code></pre></li> <li>Fields which may contain lists of values (e.g., <code>creator</code>, <code>contributor</code>) now always   returns a list (which may also be empty or contain a single element).</li> <li>Fields without a set value are no longer automatically removed from the response.</li> </ul>"},{"location":"apiv2/migration/#get-datalistfilters","title":"<code>GET /data/list/{filters}</code>","text":"<p>The endpoint now accepts the filters in the body of the request, instead of as query parameters. <pre><code>-  curl -d '' 127.0.0.1:8002/api/v1/json/data/list/status/active\n+ curl -X 'POST' 'http://localhost:8001/v1/datasets/list' \\\n+  -H 'Content-Type: application/json' \\\n+  -d '{}'\n</code></pre> This endpoint is now also available via a <code>POST</code> request, and will exhibit the same behavior regardless of how it is accessed.</p> <p>When accessing this endpoint when authenticated as administrator, it now correctly includes datasets which are private.</p> <p>The <code>limit</code> and <code>offset</code> parameters can now be used independently, you no longer need to provide both if you wish to set only one.</p>"},{"location":"apiv2/migration/#post-datasetstag","title":"<code>POST /datasets/tag</code>","text":"<p>When successful, the \"tag\" property in the returned response is now always a list, even if only one tag exists for the entity. For example, after tagging dataset 21 with the tag <code>\"foo\"</code>: <pre><code>{\n   data_tag\": {\n      \"id\": \"21\",\n-      \"tag\": \"foo\"\n+      \"tag\": [\"foo\"]\n   }\n}\n</code></pre></p>"},{"location":"apiv2/migration/#studies","title":"Studies","text":""},{"location":"apiv2/migration/#get-id_or_alias","title":"<code>GET /{id_or_alias}</code>","text":"<p>Old-style \"legacy\" studies which are solely based on tags are no longer supported.</p> Affected Legacy Studies <p>Only 24 old studies were affected by this change, listed below. There is currently not yet a migration plan for these studies.</p> id name 1 A large-scale comparison of classification algorit... 2 Fast Algorithm Selection using Learning Curves 3 Multi-Task Learning with a Natural Metric for Quan... 5 Local and Global Feature Selection on Multilabel T... 7 Massive machine learning experiments using mlr and... 8 Decision tree comparaison 10 Collaborative primer 11 Having a Blast: Meta-Learning and Heterogeneous En... 12 Subspace Clustering via Seeking Neighbors with Min... 13 Meta-QSAR: learning how to learn QSARs 17 Subgroup Discovery 20 Mythbusting data mining urban legends through larg... 22 Identifying critical paths in undergraduate progra... 24 OpenML R paper 25 Bernd Demo Study for Multiclass SVMs OML WS 2016 27 Compare three different SVM versions of R package ... 30 OpenML Paper Study 31 Iris Data set Study 32 Data Streams and more 34 Massively Collaborative Machine Learning 37 Speeding up Algorithm Selection via Meta-learning ... 38 Performance of new ctree implementations on classi... 41 ASLib OpenML Scenario 50 Hyper-parameter tuning of Decision Trees 51 ensemble on diabetes"},{"location":"apiv2/migration/#flows","title":"Flows","text":""},{"location":"apiv2/migration/#get-flowexistsnameexternal_version","title":"<code>GET /flow/exists/{name}/{external_version}/</code>","text":"<p>Behavior has changed slightly. When a flow is found:</p> <pre><code>- { \"flow_exists\": { \"exists\": \"true\", \"flow_id\": \"123\" } }\n+ { \"flow_id\": 123 }\n</code></pre> <p>and when a flow is not found: <pre><code>- { \"flow_exists\": { \"exists\": \"false\", \"flow_id\": \"-1\" } }\n+ { \"detail\": \"Flow not found.\" }\n</code></pre> and the HTTP header status code is <code>404</code> (NOT FOUND) instead of <code>200</code> (OK).</p> <p>In the future the successful case will more likely just return the flow immediately instead (see #170).</p>"},{"location":"apiv2/migration/#others","title":"Others","text":""},{"location":"apiv2/migration/#get-estimationprocedurelist","title":"<code>GET /estimationprocedure/list</code>","text":"<p>The <code>ttid</code> field has been renamed to <code>task_type_id</code>. All values are now typed. Outer levels of nesting have been removed.</p>"},{"location":"apiv2/migration/#get-evaluationmeasureslist","title":"<code>GET /evaluationmeasures/list</code>","text":"<p>Outer levels of nesting have been removed.</p>"},{"location":"apiv2/contributing/","title":"Contributing","text":"<p>Like many other open source projects, contributions from the community are an essential piece in making OpenML successful. We very much appreciate that you want to make a contribution. To try to make this as smooth as possible for everyone involved, we wrote this contribution guide. There are many kinds of contributions, such as bug reports, participating in discussions, making code contributions, or updating documentation.</p>"},{"location":"apiv2/contributing/#bug-reports","title":"Bug reports","text":"<p>When you encounter an issue using or deploying the Python-based REST API, the first step is to search our issue tracker to see if others have already reported the issue. If this is the case, please indicate that you encountered the issue by leaving a  on the issue. Feel free to leave a comment if you have additional information that may be useful to track down or fix the bug.</p> <p>If the bug has not been reported before, please open a new issue to report it. When you open a bug report, please include:</p> <ul> <li>a minimal reproducible example,</li> <li>a description of the expected behavior,</li> <li>a description of the encountered behavior,</li> <li>and any additional information that you think may be relevant, such as which environment you encountered it in, or when you first encountered the bug.</li> </ul>"},{"location":"apiv2/contributing/#code","title":"Code","text":"<p>If you want to make code contributions, please first make sure there is an open issue that describes what the contribution should look like. This may be a proposal that you write yourself, or an existing open issue that you want to help us with. Please indicate on that issue that you would like to contribute to it. This way, we can ensure that everything is clear, it's not out of date, and \"officially\" assign you to the issue so that others know its being worked on. Making sure there is a clear description and clear assignment helps prevent a situation where someone makes a large contribution that is unwanted, or is simultaneously developed by someone else. With an issue assigned, please head over to the \"Development\" section.</p>"},{"location":"apiv2/contributing/#documentation","title":"Documentation","text":"<p>For minor fixes, it's fine to make the changes and submit them through a pull request. For larger changes, please make sure you are assigned to an issue first as is described in the \"Code\" section of this page. Then, visit the \"Documentation\" page to learn how to contribute documentation changes.</p>"},{"location":"apiv2/contributing/#other","title":"Other","text":"<p>If you are looking to contribute to OpenML in general, visit \"Contributing\" on the OpenML website.</p>"},{"location":"apiv2/contributing/contributing/","title":"Setting up the development environment","text":"<p>First, follow the installation instructions for contributors to install a local fork with optional development dependencies. Stop when you reach the section \"Setting up a Database Server\".</p>"},{"location":"apiv2/contributing/contributing/#pre-commit","title":"Pre-commit","text":"<p>We use <code>pre-commit</code> to ensure certain tools and checks are ran before each commit. These tools perform code-formatting, linting, and more. This makes the code more consistent across the board, which makes it easier to work with each other's code and also can catch common errors. After installing it, it will automatically run when you try to make a commit. Install it now and verify that all checks pass out of the box:</p> <p>Install pre-commit and verify it works<pre><code>pre-commit install\npre-commit run --all-files\n</code></pre> Running the tool the first time may be slow as tools need to be installed, and many tools will build a cache so subsequent runs are faster. Under normal circumstances, running the pre-commit chain shouldn't take more than a few seconds.</p>"},{"location":"apiv2/contributing/contributing/#docker","title":"Docker","text":"<p>With the projected forked, cloned, and installed, the easiest way to set up all required services for development is through <code>docker compose</code>.</p>"},{"location":"apiv2/contributing/contributing/#starting-containers","title":"Starting containers","text":"<pre><code>docker compose up\n</code></pre> <p>This will spin up 4 containers, as defined in the <code>docker-compose.yaml</code> file:</p> <ul> <li><code>openml-test-database</code>: this is a mysql database prepopulated with test data.     It is reachable from the host machine with port <code>3306</code>, by default it is configured     to have a root user with password <code>\"ok\"</code>.</li> <li><code>server-api-docs-1</code>: this container serves project documentation at <code>localhost:8000</code>.     These pages are built from the documents in the <code>docs/</code> directory of this repository,     whenever you edit and save a file there, the page will immediately be updated.</li> <li><code>server-api-php-api-1</code>: this container serves the old PHP REST API at <code>localhost:8002</code>.     For example, visit http://localhost:8002/api/v1/json/data/1     to fetch a JSON description of dataset 1.</li> <li><code>python-api</code>: this container serves the new Python-based REST API at <code>localhost:8001</code>.     For example, visit http://localhost:8001/docs to see     the REST API documentation. Changes to the code in <code>src/</code> will be reflected in this     container.</li> </ul> <p>You don't always need every container, often just having a database and the Python-based REST API may be enough. In that case, only specify those services:</p> <pre><code>docker compose up database python-api\n</code></pre> <p>Refer to the <code>docker compose</code> documentation for more uses.</p>"},{"location":"apiv2/contributing/contributing/#connecting-to-containers","title":"Connecting to containers","text":"<p>To connect to a container, run:</p> <pre><code>docker exec -it CONTAINER_NAME /bin/bash\n</code></pre> <p>where <code>CONTAINER_NAME</code> is the name of the container. If you are unsure of your container name, then <code>docker container ls</code> may help you find it. Assuming the default container names are used, you may connect to the Python-based REST API container using:</p> <pre><code>docker exec -it python-api /bin/bash\n</code></pre> <p>This is useful, for example, to run unit tests in the container:</p> <pre><code>python -m pytest -x -v -m \"not web\"\n</code></pre>"},{"location":"apiv2/contributing/contributing/#unit-tests","title":"Unit tests","text":"<p>Our unit tests are written with the <code>pytest</code> framework. An invocation could look like this:</p> <pre><code>python -m pytest -v -x --lf -m \"not web\"\n</code></pre> <p>Where <code>-v</code> show the name of each test ran, <code>-x</code> ensures testing stops on first failure, <code>--lf</code> will first run the test(s) which failed last, and <code>-m \"not web\"</code> specifies which tests (not) to run.</p> <p>The directory structure of our tests follows the structure of the <code>src/</code> directory. For files, we follow the convention of appending <code>_test</code>. Try to keep tests as small as possible, and only rely on database and/or web connections when absolutely necessary.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/contributing/contributing/#yaml-validation","title":"YAML validation","text":"<p>The project contains various <code>yaml</code> files, for example to configure <code>mkdocs</code> or to describe Github workflows. For these <code>yaml</code> files we can configure automatic schema validation, to ensure that the files are valid without having to run the server. This also helps with other IDE features like autocomplete. Setting this up is not required, but incredibly useful if you do need to edit these files. The following <code>yaml</code> files have schemas:</p> File(s) Schema URL mkdocs.yml https://squidfunk.github.io/mkdocs-material/schema.json .pre-commit-config.yaml https://json.schemastore.org/pre-commit-config.json .github/workflows/*.yaml https://json.schemastore.org/github-workflow PyCharmVSCode <p>In PyCharm, these can be configured from <code>settings</code> &gt; <code>languages &amp; frameworks</code> &gt; <code>Schemas and DTDs</code> &gt; <code>JSON Schema Mappings</code>. There, add mappings per file or file pattern.</p> <p>In VSCode, these can be configured from <code>settings</code> &gt; <code>Extetions</code> &gt; <code>JSON</code> &gt; <code>Edit in settings.json</code>. There, add mappings per file or file pattern. For example:</p> <pre><code>\"json.schemas\": [\n   {\n         \"fileMatch\": [\n            \"/myfile\"\n         ],\n         \"url\": \"schemaURL\"\n   }\n\n]\n</code></pre>"},{"location":"apiv2/contributing/contributing/#connecting-to-another-database","title":"Connecting to another database","text":"<p>In addition to the database setup described in the installation guide, we also host a database on our server which may be connected to that is available to OpenML core contributors. If you are a core contributor and need access, please reach out to one of the engineers in Eindhoven.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/contributing/documentation/","title":"Documentation with mkdocs-material","text":"<p>Our documentation is built using mkdocs and the mkdocs-material theme. We also use some plugins. Please refer to the \"Getting Started\" pages of <code>mkdocs-material</code> for a general overview on how to work with <code>mkdocs-material</code>. All documentation files are in the <code>docs/</code> folder, except for the configuration file which is <code>mkdocs.yml</code> at the root of the repository.</p> <p>For minor changes, it should be fine to edit the page directly on Github. That should commit to a separate branch (or fork), and you can set up a pull request. For larger changes, clone a fork of the repository as described in the \"Local Installation\" section.</p> DockerLocal installation <p>After cloning the repository, you may also build and serve the documentation through Docker:</p> <pre><code>docker compose up docs\n</code></pre> <p>Instead of installing all dependencies (with <code>python -m pip install -e \".[docs]\"</code>), you may also install just the documentation dependencies:</p> <pre><code>python -m pip install mkdocs-material mkdocs-section-index\n</code></pre> <p>You can then build and serve the documentation with</p> <pre><code>python -m mkdocs serve\n</code></pre> <p>This will serve the documentation from the <code>docs/</code> directory to http://localhost:8000/. Any updates you make to files in that directory will be reflected on the website. When you are happy with your changes, just commit and set up a pull request!</p>"},{"location":"apiv2/contributing/project_overview/","title":"Project overview","text":"<p>The Python-based REST API serves several groups of endpoints:</p> <ul> <li><code>/old/</code>: serves the old-style JSON format, this should mimic the PHP responses exactly with the only deviations recorded in the migration guide.</li> <li><code>/mldcat_ap/</code>: serves datasets in MLDCAT_AP format.</li> <li><code>/*</code>: serves new-style JSON format. At this point it is intentionally similar to the old-style format.</li> </ul> <p>The endpoints are specified in subdirectories of <code>src/routers</code>. They pull data from the database through the <code>src/database</code> module. The schemas for each entity, and possible conversions between them, are defined in the <code>src/schemas</code> directory.</p> <p>Instructions are incomplete. Please have patience while we are adding more documentation.</p>"},{"location":"apiv2/contributing/tests/","title":"Writing Tests","text":"<p>tl;dr:  - Setting up the <code>py_api</code> fixture to test directly against a REST API endpoint is really slow, only use it for migration/integration tests.  - Getting a database fixture and doing a database call is slow, consider mocking if appropriate.</p>"},{"location":"apiv2/contributing/tests/#overhead-from-fixtures","title":"Overhead from Fixtures","text":"<p>Sometimes, you want to interact with the REST API through the <code>py_api</code> fixture, or want access to a database with <code>user_test</code> or <code>expdb_test</code> fixtures. Be warned that these come with considerable relative overhead, which adds up when running thousands of tests.</p> <pre><code>@pytest.mark.parametrize('execution_number', range(5000))\ndef test_private_dataset_owner_access(\n        execution_number,\n        expdb_test: Connection,\n        user_test: Connection,\n        py_api: TestClient,\n) -&gt; None:\n    fetch_user(ApiKey.REGULAR_USER, user_test)  # accesses only the user db\n    get_estimation_procedures(expdb_test)  # accesses only the experiment db\n    py_api.get(\"/does/not/exist\")  # only queries the api\n    pass\n</code></pre> <p>When individually adding/removing components, we measure (for 5000 repeats, n=1):</p> expdb user api exp call user call api get time (s) \u274c \u274c \u274c \u274c \u274c \u274c 1.78 \u2705 \u274c \u274c \u274c \u274c \u274c 3.45 \u274c \u2705 \u274c \u274c \u274c \u274c 3.22 \u274c \u274c \u2705 \u274c \u274c \u274c 298.48 \u2705 \u2705 \u274c \u274c \u274c \u274c 4.44 \u2705 \u2705 \u2705 \u274c \u274c \u274c 285.69 \u2705 \u274c \u274c \u2705 \u274c \u274c 4.91 \u274c \u2705 \u274c \u274c \u2705 \u274c 5.81 \u2705 \u2705 \u2705 \u2705 \u2705 \u2705 307.91 <p>Adding a fixture that just returns some value adds only minimal overhead (1.91s), so the burden comes from establishing the database connection itself.</p> <p>We make the following observations:</p> <ul> <li>Adding a database fixture adds the same overhead as instantiating an entirely new test.</li> <li>Overhead of adding multiple database fixtures is not free</li> <li>The <code>py_api</code> fixture adds two orders of magnitude more overhead</li> </ul> <p>We want our tests to be fast, so we want to avoid using these fixtures when we reasonably can. We restrict usage of <code>py_api</code> fixtures to integration/migration tests, since it is very slow. These only run on CI before merges. For database fixtures</p> <p>We will write some fixtures that can be used to e.g., get a <code>User</code> without accessing the database. The validity of these users will be tested against the database in only a single test.</p>"},{"location":"apiv2/contributing/tests/#mocking","title":"Mocking","text":"<p>Mocking can help us reduce the reliance on database connections in tests. A mocked function can prevent accessing the database, and instead return a predefined value instead.</p> <p>It has a few upsides:  - It's faster than using a database fixture (see below).  - The test is not dependent on the database: you can run the test without a database.</p> <p>But it also has downsides:  - Behavior changes in the database, such as schema changes, are not automatically reflected in the tests.  - The database layer (e.g., queries) are not actually tested.</p> <p>Basically, the mocked behavior may not match real behavior when executed on a database. For this reason, for each mocked entity, we should add a test that verifies that if the database layer is invoked with the database, it returns the expected output that matches the mock. This is additional overhead in development, but hopefully it pays back in more granular test feedback and faster tests.</p> <p>On the speed of mocks, consider these two tests:</p> <p><pre><code>@pytest.mark.parametrize('execution_number', range(5000))\ndef test_private_dataset_owner_access(\n        execution_number,\n        admin,\n+        mocker,\n-        expdb_test: Connection,\n) -&gt; None:\n+    mock = mocker.patch('database.datasets.get')\n+    class Dataset(NamedTuple):\n+        uploader: int\n+        visibility: Visibility\n+    mock.return_value = Dataset(uploader=1, visibility=Visibility.PRIVATE)\n\n    _get_dataset_raise_otherwise(\n        dataset_id=1,\n        user=admin,\n-        expdb=expdb_test,\n+        expdb=None,\n    )\n</code></pre> There is only a single database call in the test. It fetches a record on an indexed field and does not require any joins. Despite the database call being very light, the database-included test is ~50% slower than the mocked version (3.50s vs 5.04s).</p>"},{"location":"benchmark/","title":"Benchmarking suites","text":"<p>Machine learning research depends on objectively interpretable, comparable, and reproducible algorithm benchmarks. OpenML aims to facilitate the creation of curated, comprehensive suites of machine learning tasks, covering precise sets of conditions.</p> <p>Seamlessly integrated into the OpenML platform, benchmark suites standardize the setup, execution, analysis, and reporting of benchmarks. Moreover, they make benchmarking a whole lot easier:  </p> <ul> <li> <p>all datasets are uniformly formatted in standardized data formats </p> </li> <li> <p>they can be easily downloaded programmatically through APIs and client libraries</p> </li> <li> <p>they come with machine-readable meta-information, such as the occurrence of missing values, to train algorithms correctly</p> </li> <li> <p>standardized train-test splits are provided to ensure that results can be objectively compared</p> </li> <li> <p>results can be shared in a reproducible way through the APIs</p> </li> <li> <p>results from other users can be easily downloaded and reused</p> </li> </ul>"},{"location":"benchmark/#software-interfaces","title":"Software interfaces","text":"<p>To use OpenML Benchmark suites, you can use bindings in several programming languages. These all interface with the OpenML REST API. The default endpoint for this is <code>https://www.openml.org/api/v1/</code>, but this can change when later versions of the API are released. To use the code examples below, you only need a recent version of one of the following libraries:</p> <ul> <li>OpenML Java ApiConnector (version <code>1.0.22</code> and up).</li> <li>OpenML Weka (version <code>0.9.6</code> and up). This package adds a Weka Integration.</li> <li>OpenML Python (version <code>0.9.0</code> and up)</li> <li>OpenML R (version <code>1.8</code> and up)</li> </ul>"},{"location":"benchmark/#using-openml-benchmark-suites","title":"Using OpenML Benchmark Suites","text":"<p>Below are walk-through instructions for common use cases, as well as code examples. These illustrations use the reference OpenML-CC18 benchmark suite, but you can replace it with any other benchmark suite. Note that a benchmark suite is a set of OpenML <code>tasks</code>, which envelop not only a specific dataset, but also the train-test splits and (for predictive tasks) the target feature.</p> Terminology and current status <p>Benchmark suites are sets of OpenML tasks that you can create and manage yourself. At the same time, it is often useful to also share the set of experiments (runs) with the ensuing benchmarking results. For legacy reasons, such sets of tasks or runs are called <code>studies</code> in the OpenML REST API. In the OpenML bindings (Python, R, Java,...) these are called either <code>sets</code> or <code>studies</code>.</p> <p>When benchmarking, you will probably use two types of sets:</p> <ul> <li>Sets of tasks. These can be created, edited, downloaded or deleted via the OpenML API. Website forms will be added soon. Also the set of underlying datasets can be easily retrieved via the API.</li> <li>Sets of runs. Likewise, these can be created, edited, downloaded or deleted via the OpenML API. On the website, these are currently simply called 'studies'. Also the set of underlying tasks, datasets and flows can be easily retrieved. It is possible to link a set of runs to a benchmark study, aimed to collect future runs on that specific set of tasks. Additional information on these will be provided in a separate page.</li> </ul>"},{"location":"benchmark/#listing-the-benchmark-suites","title":"Listing the benchmark suites","text":"<p>The current list of benchmark suites is explicitly listed on the bottom of this page. The list of all sets of tasks can also be fetched programmatically. This list includes the suite's ID (and optionally an alias), which can be used to fetch further details.</p> <p>Via the REST API, the list is returned in XML or JSON</p> REST <p>https://www.openml.org/api/v1/xml/study/list/main_entity_type/task/status/all</p> <p>Check out the API docs</p> Python example <pre><code>import openml\n\n# using the main entity type task, only benchmark suites are returned\n# each benchmark suite has an ID, some also have an alias. These can be\n# used to obtain the full details. \nstudies = openml.study.list_suites(status = 'all')\n</code></pre> Java example <pre><code>public void listBenchmarksuites() throws Exception {\n    OpenmlConnector openml = new OpenmlConnector();\n    Map&lt;String, String&gt; filters = new TreeMap&lt;String, String&gt;();\n    filters.put(\"status\", \"all\");\n    filters.put(\"main_entity_type\", \"task\");\n    filters.put(\"limit\", \"20\");\n    StudyList list = openml.studyList(filters);\n}\n</code></pre> R example <pre><code>studies = listOMLStudies()\n</code></pre>"},{"location":"benchmark/#fetching-details","title":"Fetching details","text":"<p>Using the ID or alias of a benchmark suite, you can retrieve a description and the full list of tasks and the underlying datasets.</p> <p>Via the REST API, a list of all tasks and dataset IDs is returned in XML or JSON</p> REST <p>https://www.openml.org/api/v1/xml/study/OpenML-CC18</p> <p>Check out the API docs</p> <p>In Python, the data is returned as <code>features, targets</code> numpy arrays:</p> Python example <pre><code>import openml\n\nbenchmark_suite = openml.study.get_suite('OpenML-CC18') # obtain the benchmark suite\n\nfor task_id in benchmark_suite.tasks:  # iterate over all tasks\n    task = openml.tasks.get_task(task_id)  # download the OpenML task\n    features, targets = task.get_X_and_y()  # get the data\n</code></pre> <p>In Java, the data is returned as a WEKA Instances object:</p> Java example <pre><code>public void downloadDatasets() throws Exception {\n    OpenmlConnector openml = new OpenmlConnector();\n    Study benchmarksuite = openml.studyGet(\"OpenML-CC18\", \"tasks\");\n    for (Integer taskId : benchmarksuite.getTasks()) { // iterate over all tasks\n        Task t = openml.taskGet(taskId); // download the OpenML task\n        // note that InstanceHelper is part of the OpenML-weka package\n        Instances d = InstancesHelper.getDatasetFromTask(openml, t); // obtain the dataset\n    }\n}\n</code></pre> <p>In R, the data is returned as an R dataframe:</p> R example <pre><code>library(OpenML)\ntask.ids = getOMLStudy('OpenML-CC18')$tasks$task.id # obtain the list of suggested tasks\nfor (task.id in task.ids) { # iterate over all tasks\n  task = getOMLTask(task.id) # download single OML task\n  data = as.data.frame(task) # obtain raw data set\n</code></pre>"},{"location":"benchmark/#running-and-sharing-benchmarks","title":"Running and sharing benchmarks","text":"<p>The code below demonstrates how OpenML benchmarking suites can be conveniently imported for benchmarking using the Python, Java and R APIs.</p> <p>First, the list of tasks is downloaded as already illustrated above. Next, a specific algorithm (or pipeline) can be run on each of them. The OpenML API will automatically evaluate the algorithm using the pre-set train-test splits and store the predictions and scores in a run object. This run object can then be immediately published, pushing the results to the OpenML server, so that they can be compared against all others on the same benchmark set. Uploading results requires an OpenML API key, which can be found in your account details after logging into the OpenML website.</p> REST <p>Requires POST requests: Attaching a new run to a benchmark_study Detaching a run from benchmark_study </p> Python example <pre><code>import openml\nimport sklearn\n\nopenml.config.apikey = 'FILL_IN_OPENML_API_KEY'  # set the OpenML Api Key\nbenchmark_suite = openml.study.get_suite('OpenML-CC18')  # obtain the benchmark suite\n\n# build a scikit-learn classifier\nclf = sklearn.pipeline.make_pipeline(sklearn.preprocessing.Imputer(),\n                                     sklearn.tree.DecisionTreeClassifier())\n\nfor task_id in benchmark_suite.tasks:  # iterate over all tasks\n\n    task = openml.tasks.get_task(task_id)  # download the OpenML task\n    run = openml.runs.run_model_on_task(clf, task)  # run the classifier on the task\n    score = run.get_metric_fn(sklearn.metrics.accuracy_score)  # print accuracy score\n    print('Data set: %s; Accuracy: %0.2f' % (task.get_dataset().name,score.mean()))\n    run.publish()  # publish the experiment on OpenML (optional, requires internet and an API key)\n    print('URL for run: %s/run/%d' %(openml.config.server,run.run_id))\n</code></pre> Java example <pre><code>public static void runTasksAndUpload() throws Exception {\n  OpenmlConnector openml = new OpenmlConnector();\n  openml.setApiKey(\"FILL_IN_OPENML_API_KEY\");\n  // obtain the benchmark suite\n  Study benchmarksuite = openml.studyGet(\"OpenML-CC18\", \"tasks\");\n  Classifier tree = new REPTree(); // build a Weka classifier\n  for (Integer taskId : benchmarksuite.getTasks()) { // iterate over all tasks\n    Task t = openml.taskGet(taskId); // download the OpenML task\n    Instances d = InstancesHelper.getDatasetFromTask(openml, t); // obtain the dataset\n    int runId = RunOpenmlJob.executeTask(openml, new WekaConfig(), taskId, tree);\n    Run run = openml.runGet(runId);   // retrieve the uploaded run\n  }\n}\n</code></pre> R example <pre><code>library(OpenML)\nsetOMLConfig(apikey = 'FILL_IN_OPENML_API_KEY')\nlrn = makeLearner('classif.rpart') # construct a simple CART classifier\ntask.ids = getOMLStudy('OpenML-CC18')$tasks$task.id # obtain the list of suggested tasks\nfor (task.id in task.ids) { # iterate over all tasks\n  task = getOMLTask(task.id) # download single OML task\n  data = as.data.frame(task) # obtain raw data set\n  run = runTaskMlr(task, learner = lrn) # run constructed learner\n  upload = uploadOMLRun(run) # upload and tag the run\n}\n</code></pre>"},{"location":"benchmark/#retrieving-runs-on-a-benchmarking-suites","title":"Retrieving runs on a benchmarking suites:","text":"<p>Once a benchmark suite has been created, the listing functions can be used to  obtain all results on the benchmark suite. Note that there are several other ways to select and bundle runs together. This will be featured in  a separate article on reproducible benchmarks. </p> REST (TODO) <p>https://www.openml.org/api/v1/xml/run/list/study/OpenML-CC18</p> <p>Check out the API docs</p> Python example <pre><code>benchmark_suite = openml.study.get_suite('OpenML-CC18')\nruns = openml.runs.list_runs(task=benchmark_suite.tasks, limit=1000)\n</code></pre> Java example <pre><code>public void downloadResultsBenchmarkSuite()  throws Exception {\n    Study benchmarkSuite = openml.studyGet(\"OpenML100\", \"tasks\");\n\n    Map&lt;String, List&lt;Integer&gt;&gt; filters = new TreeMap&lt;String, List&lt;Integer&gt;&gt;();\n    filters.put(\"task\", Arrays.asList(benchmarkSuite.getTasks()));\n    RunList rl = openml.runList(filters, 200, null);\n\n    assertTrue(rl.getRuns().length &gt; 0); \n}\n</code></pre> R example <pre><code>benchmark.suite = getOMLStudy(study = \"OpenML-CC18\")\nrun.ids = extractOMLStudyIds(benchmark.suite, \"run.id\")\nruns = rbindlist(lapply(run.ids, function(id) listOMLRuns(run.id = id)))\n# TODO waiting for REST API\n</code></pre>"},{"location":"benchmark/#creating-new-benchmark-suites","title":"Creating new benchmark suites","text":"<p>Additional OpenML benchmark suites can be created by defining the precise set of tasks, as well as a textual description. New datasets first need to be registered on OpenML and tasks need to be created on them.</p> <p>We have provided a GitHub repository with additional tools and scripts to build new benchmark studies, e.g. to select all datasets adhering to strict conditions, and to analyse bencharking results.</p> REST <p>Requires POST requests: Creating a benchmark suite </p> Python example <pre><code>import openml\n\n# find 250 tasks that we are interested in, e.g., the tasks that have between\n# 100 and 10000 instances and between 4 and 20 attributes\ntasks = openml.tasks.list_tasks(number_instances='100..10000', number_features='4..20', size=250)\ntask_ids = list(tasks.keys())\n\n# create the benchmark suite\n# the arguments are the alias, name, description, and list of task_ids, respectively.\nstudy = openml.study.create_benchmark_suite( \n    name=\"MidSize Suite\", \n    alias=None,\n    description=\"illustrating how to create a benchmark suite\", \n    task_ids=task_ids,\n)\nstudy_id = study.publish()\n</code></pre> Java example <pre><code>public void createBenchmarkSuite() throws Exception {\n    OpenmlConnector openml = new OpenmlConnector(\"FILL_IN_OPENML_API_KEY\");\n    // find 250 tasks that we are interested in, e.g., the tasks that have between\n    // 100 and 10000 instances and between 4 and 20 attributes\n    Map&lt;String, String&gt; filtersOrig = new TreeMap&lt;String, String&gt;();\n    filtersOrig.put(\"number_instances\", \"100..10000\");\n    filtersOrig.put(\"number_features\", \"4..20\");\n    filtersOrig.put(\"limit\", \"250\");\n    Tasks tasksOrig = client_write_test.taskList(filtersOrig);\n\n    // create the study\n    Study study = new Study(null, \"test\", \"test\", null, tasksOrig.getTaskIds(), null);\n    int studyId = openml.studyUpload(study);\n}\n</code></pre> R example <pre><code># find 250 tasks with 100 and 10000 instances and between 4 and 20 attributes\ntid = listOMLTasks(number.of.instances = c(100, 10000), number.of.features = c(4, 20), limit = 250)\nstudy = makeOMLStudy(alias = \"test_alias\", name = \"Test Upload from R\", description = \"Just testing\", task.id = tid$task.id)\nid = uploadOMLStudy(study)\n</code></pre>"},{"location":"benchmark/#updating-a-benchmark-suite","title":"Updating a benchmark suite","text":"<p>You can add tasks to a benchmark suite, or remove them.</p> REST <p>Requires POST requests: Attaching a new task Detaching a task </p> Python example <pre><code>import openml\n\n# find 250 tasks that we are interested in, e.g., the tasks that have between\n# 100 and 10000 instances and between 4 and 20 attributes\ntasks = openml.tasks.list_tasks(number_instances='100..10000', number_features='4..20', size=250)\ntask_ids = list(tasks.keys())\n\n# create the benchmark suite\nstudy = openml.study.create_benchmark_suite( \n    name=\"MidSize Suite\", \n    alias=None,\n    description=\"illustrating how to create a benchmark suite\", \n    task_ids=task_ids,\n)\nstudy_id = study.publish()\n\n# download the study from the server, for verification purposes\nstudy = openml.study.get_study(study_id)\n\n# until the benchmark suite is activated, we can also add some more tasks. Search for the letter dataset:\ntasks_new = openml.tasks.list_tasks(data_name='letter', size=1)\ntask_ids_new = list(tasks_new.keys())\nopenml.study.attach_to_study(study_id, task_ids_new)\n\n# or even remove these again\nopenml.study.detach_from_study(study_id, task_ids_new)\n\n# redownload the study\nstudy_prime = openml.study.get_study(study_id)\n\nassert(study.tasks == study_prime.tasks)\nassert(study.data == study_prime.data)\n</code></pre> Java example <pre><code>public void attachDetachStudy()  throws Exception {\n    OpenmlConnector openml = new OpenmlConnector(\"FILL_IN_OPENML_API_KEY\");\n    // find 250 tasks that we are interested in, e.g., the tasks that have between\n    // 100 and 10000 instances and between 4 and 20 attributes\n    Map&lt;String, String&gt; filtersOrig = new TreeMap&lt;String, String&gt;();\n    filtersOrig.put(\"number_instances\", \"100..10000\");\n    filtersOrig.put(\"number_features\", \"4..20\");\n    filtersOrig.put(\"limit\", \"250\");\n    Tasks tasksOrig = openml.taskList(filtersOrig);\n\n    // create the study\n    Study study = new Study(null, \"test\", \"test\", null, tasksOrig.getTaskIds(), null);\n    int studyId = openml.studyUpload(study);\n\n    // until the benchmark suite is activated, we can also add some more tasks. Search for the letter dataset:\n    Map&lt;String, String&gt; filtersAdd = new TreeMap&lt;String, String&gt;();\n    filtersAdd.put(\"data_name\", \"letter\");\n    filtersAdd.put(\"limit\", \"1\");\n    Tasks tasksAdd = openml.taskList(filtersAdd);\n    openml.studyAttach(studyId, Arrays.asList(tasksAdd.getTaskIds()));\n\n    // or even remove these again\n    openml.studyDetach(studyId, Arrays.asList(tasksAdd.getTaskIds()));\n\n    // download the study\n    Study studyDownloaded = openml.studyGet(studyId);\n    assertArrayEquals(tasksOrig.getTaskIds(), studyDownloaded.getTasks());\n}\n</code></pre> R example <pre><code>TODO\n</code></pre>"},{"location":"benchmark/#further-code-examples-and-use-cases","title":"Further code examples and use cases","text":"<p>As mentioned above, we host a GitHub repository with additional tools and scripts to easily create and use new benchmark studies. It includes:</p> <ul> <li>A Jupyter Notebook that builds a new benchmark suite with datasets that adhere to strict and complex conditions, as well as automated tests to remove tasks that are too easy for proper benchmarking.</li> <li>A Jupyter Notebook that shows how to pull in the latest state-of-the-art results for any of the benchmark suites</li> <li>A Jupyter Notebook that does a detailed analysis of all results in a benchmark suite, and an example run on the OpenML-CC18. It includes a wide range of plots and rankings to get a deeper insight into the benchmark results.</li> <li>Scripts in Python and R to facilitate common subtasks.</li> </ul> <p>We very much welcome new scripts and notebooks, or improvements to the existing ones, that help others to create benchmark suites and analyse benchmarking results.</p>"},{"location":"benchmark/#list-of-benchmarking-suites","title":"List of benchmarking suites","text":""},{"location":"benchmark/#openml-cc18","title":"OpenML-CC18","text":"<p>The OpenML-CC18 suite contains all OpenML datasets from mid-2018 that satisfy a large set of clear requirements for thorough yet practical benchmarking. It includes datasets frequently used in benchmarks published over the last years, so it can be used as a drop-in replacement for many benchmarking setups.</p> <p>List of datasets and properties</p> <p>The suite is defined as the set of all verified OpenML datasets that satisfy the following requirements:</p> <ul> <li>the number of observations are between 500 and 100000 to focus on medium-sized datasets, that are not too small and not too big,</li> <li>the number of features does not exceed 5000 features to keep the runtime of algorithms low,</li> <li>the target attribute has at least two classes</li> <li>the ratio of the minority class and the majority class is above 0.05, to eliminate highly imbalanced datasets which require special treatment for both algorithms and evaluation measures.</li> </ul> <p>We excluded datasets which:</p> <ul> <li>are artificially generated (not to confuse with simulated)</li> <li>cannot be randomized via a 10-fold cross-validation due to grouped samples or because they are time series or data streams</li> <li>are a subset of a larger dataset</li> <li>have classes with less than 20 observations</li> <li>have no source or reference available</li> <li>can be perfectly classified by a single attribute or a decision stump</li> <li>allow a decision tree to achieve 100% accuracy on a 10-fold cross-validation task</li> <li>have more than 5000 features after one-hot-encoding categorical features</li> <li>are created by binarization of regression tasks or multiclass classification tasks, or</li> <li>are sparse data (e.g., text mining data sets)</li> </ul> Detailed motivation of these decisions <p>We chose the CC18 datasets to allow for practical benchmarking based on the characteristics that might be problematic based on our experience, and to avoid common pitfalls that may invalidate benchmark studies:  </p> <ul> <li>We used at least 500 data points to allow performing cross-validation while still having a large-enough test split.</li> <li>We limited the datasets to 100.000 data points to allow the algorithms to train machine learning models in a reasonable amount of time.</li> <li>We limited the number of features to 5000 to allow the usage of algorithms which scale unfavourably in the number of features. This limitation, together with the two limitations above aims to allow running all \u201cstandard\u201d machine learning algorithms (naive bayes, linear models, support vector machines, tree-based ensemble methods and neural networks) on the benchmark suite.</li> <li>We required each dataset to have at least two classes to be able to work in a supervised classification setting.</li> <li>We require each class to have at least 20 observations to be able to perform stratified cross-validation where there is at least one observation from each class in each split. We have found that not having all classes present in all training and test sets can make several machine learning packages fail.</li> <li>We require a certain balancedness (ratio of minority class to majority class) to prevent cases where only predicting the majority class would be beneficial. This is most likely the restriction which is most debatable, but we found it very helpful to apply a large set of machine learning algorithms across several libraries to the study. We expect that future studies focus more on imbalanced datasets. </li> </ul> <p>Furthermore, we aimed to have the dataset collection as general as possible, rule out as few algorithms as possible and have it usable as easily as possible:</p> <ul> <li>We strived to remove artificial datasets as they, for example, come from textbooks and it is hard to reliably assess their difficulty. We admit that there is a blurred line between artificial and simulated datasets and do not have a perfect distinction between them (for example, a lot of phenomena can be simulated, but the outcome might be like a simple, artificial dataset). Therefore, we removed datasets if we were in doubt of whether they are simulated or artificial. </li> <li>We removed datasets which require grouped sampling because they are time series or data streams which should be treated with special care by machine learning algorithms (i.e., taking the time aspect into account). To be on the safe side, we also removed datasets where each sample constitutes a single data stream.</li> <li>We removed datasets which are a subset of larger datasets. Allowing subsets would be very subjective as there is no objective choice of a dataset subset size or a subset of the variables or classes. Therefore, creating dataset subsets would open a Pandora\u2019s Box.</li> <li>We removed datasets which have no source or reference available to potentially learn more about these datasets if we observe unexpected behavior in future studies. In contrast, we would not be able to learn more about the background of a dataset which has no description and publication attached, leaving us with a complete black box.</li> <li>We removed datasets which can be perfectly classified by a single attribute or a decision stump as they do not allow to meaningfully compare machine learning algorithms (they all achieve 100% accuracy unless the hyperparameters are set in a bogus way).</li> <li>We removed datasets where a decision tree could achieve 100% accuracy on a 10-fold cross-validation task to remove datasets which can be solved by a simple algorithm which is prone to overfitting training data. We found that this is a good indicator of too easy datasets. Obviously, other datasets will appear easy for several algorithms, and we aim to learn more about the characteristics of such datasets in future studies.</li> <li>We removed datasets which have more than 5000 features after one-hot-encoding categorical features. One-hot-encoding is the most frequent way to deal with categorical variables across the different machine learning libraries MLR, scikit-learn and WEKA. In order to limit the number of features to 5000 as explained above, we imposed the additional constraint that this should be counted after one-hot-encoding to allow wide applicability of the benchmark suite.</li> <li>We removed datasets which were created by binarization of regression tasks or multiclass classification task for similar reasons as for forbidding dataset subsets.</li> <li>We did not include sparse datasets because not all machine learning libraries (i.e., all machine learning models) can handle them gracefully, which is in contrast to our goal which is wide applicability.</li> </ul>"},{"location":"benchmark/#citing-the-openml-cc18","title":"Citing the OpenML-CC18","text":"<p>If you have used the OpenML-CC18 in a scientific publication, we would appreciate citations of core OpenML packages as well as a citation of the following paper:</p> <p>Bischl, Bernd and Casalicchio, Giuseppe and Feurer, Matthias and Hutter, Frank and Lang, Michel and Mantovani, Rafael G. and van Rijn, Jan N. and Vanschoren, Joaquin. OpenML Benchmarking Suites. arXiv 1708.0373v2 (2019): 1-6</p>"},{"location":"benchmark/#openml100","title":"OpenML100","text":"<p>The OpenML100 was a predecessor of the OpenML-CC18, consisting of 100 classification datasets. We recommend that you use the OpenML-CC18 instead, because the OpenML100 suffers from some teething issues in the design of benchmark suites. For instance, it contains several datasets that are too easy to model with today's machine learning algorithms, as well as datasets that represent time series analysis problems. These do not invalidate benchmarks run on the OpenML100, but may obfuscate the interpretation of results. The 'OpenML-CC18' handle is also more descriptive and allows easier versioning. The OpenML100 was first published in the Arxiv preprint OpenML Benchmarking Suites and the OpenML100.</p> <p>List of datasets and properties</p> <p>For reference, the OpenML100 included datasets satisfying the following requirements:</p> <ul> <li>the number of observations are between 500 and 100000 to focus on medium-sized datasets, that are not too small for proper training and not too big for practical experimentation</li> <li>the number of features does not exceed 5000 features to keep the runtime of algorithms low</li> <li>the target attribute has at least two classes</li> <li>he ratio of the minority class and the majority class is above 0.05 to eliminate highly imbalanced datasets that would obfuscate a clear analysis</li> </ul> <p>It excluded datasets which:</p> <ul> <li>cannot be randomized via a 10-fold cross-validation due to grouped samples</li> <li>have an unknown origin or no clearly defined task</li> <li>are variants of other datasets (e.g. binarized regression tasks)</li> <li>include sparse data (e.g., text mining data sets)</li> </ul>"},{"location":"benchmark/#citing-the-openml100","title":"Citing the OpenML100","text":"<p>If you have used the OpenML100 in a scientific publication, we would appreciate citations of core OpenML packages as well as a citation of the following paper:</p> <p>Bischl, Bernd and Casalicchio, Giuseppe and Feurer, Matthias and Hutter, Frank and Lang, Michel and Mantovani, Rafael G. and van Rijn, Jan N. and Vanschoren, Joaquin. OpenML Benchmarking Suites and the OpenML100. arXiv 1708.0373v1 (2017): 1-6</p>"},{"location":"benchmark/#need-help","title":"Need help?","text":"<p>We are happy to answer to any suggestion or question you may have. For general questions or issues, please open an issue in the benchmarking issue tracker. If the issue lies with one of the language-specific bindings, please post an issue in the appropriate issue tracker.</p>"},{"location":"benchmark/automl/AutoML-Benchmark/","title":"Getting Started","text":"<p>The AutoML Benchmark is a tool for benchmarking AutoML frameworks on tabular data. It automates the installation of AutoML frameworks, passing it data, and evaluating their predictions.  Our paper describes the design and showcases  results from an evaluation using the benchmark.  This guide goes over the minimum steps needed to evaluate an AutoML framework on a toy dataset.</p> <p>Full instructions can be found in the API Documentation.</p>"},{"location":"benchmark/automl/AutoML-Benchmark/#installation","title":"Installation","text":"<p>These instructions assume that Python 3.9 (or higher)  and git are installed, and are available under the alias <code>python</code> and <code>git</code>, respectively. We recommend Pyenv for managing multiple Python installations, if applicable. We support Ubuntu 22.04, but many linux and MacOS versions likely work (for MacOS, it may be necessary to have <code>brew</code> installed).</p> <p>First, clone the repository:</p> <pre><code>git clone https://github.com/openml/automlbenchmark.git --branch stable --depth 1\ncd automlbenchmark\n</code></pre> <p>Create a virtual environments to install the dependencies in:</p>"},{"location":"benchmark/automl/AutoML-Benchmark/#linux","title":"Linux","text":"<pre><code>python -m venv venv\nsource venv/bin/activate\n</code></pre>"},{"location":"benchmark/automl/AutoML-Benchmark/#macos","title":"MacOS","text":"<pre><code>python -m venv venv\nsource venv/bin/activate\n</code></pre>"},{"location":"benchmark/automl/AutoML-Benchmark/#windows","title":"Windows","text":"<pre><code>python -m venv ./venv\nvenv/Scripts/activate\n</code></pre> <p>Then install the dependencies:</p> <pre><code>python -m pip install --upgrade pip\npython -m pip install -r requirements.txt\n</code></pre> Note for Windows users <p>The automated installation of AutoML frameworks is done using shell script, which doesn't work on Windows. We recommend you use Docker to run the examples below. First, install and run <code>docker</code>.  Then, whenever there is a <code>python runbenchmark.py ...</code>  command in the tutorial, add <code>-m docker</code> to it (<code>python runbenchmark.py ... -m docker</code>).</p> Problem with the installation? <p>On some platforms, we need to ensure that requirements are installed sequentially. Use <code>xargs -L 1 python -m pip install &lt; requirements.txt</code> to do so. If problems  persist, open an issue with the error and information about your environment (OS, Python version, pip version).</p>"},{"location":"benchmark/automl/AutoML-Benchmark/#running-the-benchmark","title":"Running the Benchmark","text":"<p>To run a benchmark call the <code>runbenchmark.py</code> script specifying the framework to evaluate.</p> <p>See the API Documentation. for more information on the parameters available.</p>"},{"location":"benchmark/automl/basic_example/","title":"Random Forest Baseline","text":"<p>Let's try evaluating the <code>RandomForest</code> baseline, which uses scikit-learn's random forest:</p>"},{"location":"benchmark/automl/basic_example/#running-the-benchmark","title":"Running the Benchmark","text":""},{"location":"benchmark/automl/basic_example/#linux","title":"Linux","text":"<pre><code>python runbenchmark.py randomforest \n</code></pre>"},{"location":"benchmark/automl/basic_example/#macos","title":"MacOS","text":"<pre><code>python runbenchmark.py randomforest \n</code></pre>"},{"location":"benchmark/automl/basic_example/#windows","title":"Windows","text":"<p>As noted above, we need to install the AutoML frameworks (and baselines) in a container. Add <code>-m docker</code> to the command as shown: <pre><code>python runbenchmark.py randomforest -m docker\n</code></pre></p> <p>Important</p> <p>Future example usages will only show invocations without <code>-m docker</code> mode, but Windows users will need to run in some non-local mode.</p>"},{"location":"benchmark/automl/basic_example/#results","title":"Results","text":"<p>After running the command, there will be a lot of output to the screen that reports on what is currently happening. After a few minutes final results are shown and should  look similar to this:</p> <pre><code>Summing up scores for current run:\n               id        task  fold    framework constraint     result      metric  duration      seed\nopenml.org/t/3913         kc2     0 RandomForest       test   0.865801         auc      11.1 851722466\nopenml.org/t/3913         kc2     1 RandomForest       test   0.857143         auc       9.1 851722467\n  openml.org/t/59        iris     0 RandomForest       test  -0.120755 neg_logloss       8.7 851722466\n  openml.org/t/59        iris     1 RandomForest       test  -0.027781 neg_logloss       8.5 851722467\nopenml.org/t/2295 cholesterol     0 RandomForest       test -44.220800    neg_rmse       8.7 851722466\nopenml.org/t/2295 cholesterol     1 RandomForest       test -55.216500    neg_rmse       8.7 851722467\n</code></pre> <p>The result denotes the performance of the framework on the test data as measured by the metric listed in the metric column. The result column always denotes performance  in a way where higher is better (metrics which normally observe \"lower is better\" are converted, which can be observed from the <code>neg_</code> prefix).</p> <p>While running the command, the AutoML benchmark performed the following steps:</p> <ol> <li>Create a new virtual environment for the Random Forest experiment.      This environment can be found in <code>frameworks/randomforest/venv</code> and will be re-used      when you perform other experiments with <code>RandomForest</code>.</li> <li>It downloaded datasets from OpenML complete with a      \"task definition\" which specifies cross-validation folds.</li> <li>It evaluated <code>RandomForest</code> on each (task, fold)-combination in a separate subprocess, where:<ol> <li>The framework (<code>RandomForest</code>) is initialized.</li> <li>The training data is passed to the framework for training.</li> <li>The test data is passed to the framework to make predictions on.</li> <li>It passes the predictions back to the main process</li> </ol> </li> <li>The predictions are evaluated and reported on. They are printed to the console and      are stored in the <code>results</code> directory. There you will find:<ol> <li><code>results/results.csv</code>: a file with all results from all benchmarks conducted on your machine.</li> <li><code>results/randomforest.test.test.local.TIMESTAMP</code>: a directory with more information about the run,     such as logs, predictions, and possibly other artifacts.</li> </ol> </li> </ol> <p>Docker Mode</p> <p>When using docker mode (with <code>-m docker</code>) a docker image will be made that contains the virtual environment. Otherwise, it functions much the same way.</p>"},{"location":"benchmark/automl/basic_example/#important-parameters","title":"Important Parameters","text":"<p>As you can see from the results above, the  default behavior is to execute a short test benchmark. However, we can specify a different benchmark, provide different constraints, and even run the experiment in a container or on AWS. There are many parameters for the <code>runbenchmark.py</code> script, but the most important ones are:</p>"},{"location":"benchmark/automl/basic_example/#framework-required","title":"Framework (required)","text":"<ul> <li>The AutoML framework or baseline to evaluate and is not case-sensitive. See   integrated frameworks for a list of supported frameworks.    In the above example, this benchmarked framework <code>randomforest</code>.</li> </ul>"},{"location":"benchmark/automl/basic_example/#benchmark-optional-defaulttest","title":"Benchmark (optional, default='test')","text":"<ul> <li>The benchmark suite is the dataset or set of datasets to evaluate the framework on.   These can be defined as on OpenML as a study or task    (formatted as <code>openml/s/X</code> or <code>openml/t/Y</code> respectively) or in a local file.   The default is a short evaluation on two folds of <code>iris</code>, <code>kc2</code>, and <code>cholesterol</code>.</li> </ul>"},{"location":"benchmark/automl/basic_example/#constraints-optional-defaulttest","title":"Constraints (optional, default='test')","text":"<ul> <li> <p>The constraints applied to the benchmark as defined by default in constraints.yaml.   These include time constraints, memory constrains, the number of available cpu cores, and more.   Default constraint is <code>test</code> (2 folds for 10 min each). </p> <p>Constraints are not enforced!</p> <p>These constraints are forwarded to the AutoML framework if possible but, except for runtime constraints, are generally not enforced. It is advised when benchmarking to use an environment that mimics the given constraints.</p> Constraints can be overriden by <code>benchmark</code> <p>A benchmark definition can override constraints on a task level. This is useful if you want to define a benchmark which has different constraints for different tasks. The default \"test\" benchmark does this to limit runtime to 60 seconds instead of 600 seconds, which is useful to get quick results for its small datasets. For more information, see defining a benchmark.</p> </li> </ul>"},{"location":"benchmark/automl/basic_example/#mode-optional-defaultlocal","title":"Mode (optional, default='local')","text":"<ul> <li> <p>The benchmark can be run in four modes:</p> <ul> <li><code>local</code>: install a local virtual environment and run the benchmark on your machine.</li> <li><code>docker</code>: create a docker image with the virtual environment and run the benchmark in a container on your machine.               If a local or remote image already exists, that will be used instead. Requires Docker.</li> <li><code>singularity</code>: create a singularity image with the virtual environment and run the benchmark in a container on your machine. Requires Singularity.</li> <li><code>aws</code>: run the benchmark on AWS EC2 instances.           It is possible to run directly on the instance or have the EC2 instance run in <code>docker</code> mode.           Requires valid AWS credentials to be configured, for more information see Running on AWS.</li> </ul> </li> </ul> <p>For a full list of parameters available, run:</p> <pre><code>python runbenchmark.py --help\n</code></pre>"},{"location":"benchmark/automl/benchmark_on_openml/","title":"Example: Benchmarks on OpenML","text":"<p>In the previous examples, we used benchmarks which were defined in a local file (test.yaml and  validation.yaml, respectively).  However, we can also use tasks and benchmarking suites defined on OpenML directly from the command line. When referencing an OpenML task or suite, we can use <code>openml/t/ID</code> or <code>openml/s/ID</code> respectively as  argument for the benchmark parameter. Running on the iris task:</p> <pre><code>python runbenchmark.py randomforest openml/t/59\n</code></pre> <p>or on the entire AutoML benchmark classification suite (this will take hours!):</p> <pre><code>python runbenchmark.py randomforest openml/s/271\n</code></pre> <p>Large-scale Benchmarking</p> <p>For large scale benchmarking it is advised to parallelize your experiments, as otherwise it may take months to run the experiments. The benchmark currently only supports native parallelization in <code>aws</code> mode (by using the <code>--parallel</code> parameter), but using the <code>--task</code> and <code>--fold</code> parameters  it is easy to generate scripts that invoke individual jobs on e.g., a SLURM cluster. When you run in any parallelized fashion, it is advised to run each process on separate hardware to ensure experiments can not interfere with each other.</p>"},{"location":"benchmark/automl/important_params/","title":"Important Parameters","text":"<p>As you can see from the results above, the  default behavior is to execute a short test benchmark. However, we can specify a different benchmark, provide different constraints, and even run the experiment in a container or on AWS. There are many parameters for the <code>runbenchmark.py</code> script, but the most important ones are:</p> <p><code>Framework (required)</code></p> <ul> <li>The AutoML framework or baseline to evaluate and is not case-sensitive. See   integrated frameworks for a list of supported frameworks.    In the above example, this benchmarked framework <code>randomforest</code>.</li> </ul> <p><code>Benchmark (optional, default='test')</code></p> <ul> <li>The benchmark suite is the dataset or set of datasets to evaluate the framework on.   These can be defined as on OpenML as a study or task    (formatted as <code>openml/s/X</code> or <code>openml/t/Y</code> respectively) or in a local file.   The default is a short evaluation on two folds of <code>iris</code>, <code>kc2</code>, and <code>cholesterol</code>.</li> </ul> <p><code>Constraints (optional, default='test')</code></p> <ul> <li> <p>The constraints applied to the benchmark as defined by default in constraints.yaml.   These include time constraints, memory constrains, the number of available cpu cores, and more.   Default constraint is <code>test</code> (2 folds for 10 min each). </p> <p>Constraints are not enforced!</p> <p>These constraints are forwarded to the AutoML framework if possible but, except for runtime constraints, are generally not enforced. It is advised when benchmarking to use an environment that mimics the given constraints.</p> Constraints can be overriden by <code>benchmark</code> <p>A benchmark definition can override constraints on a task level. This is useful if you want to define a benchmark which has different constraints for different tasks. The default \"test\" benchmark does this to limit runtime to 60 seconds instead of 600 seconds, which is useful to get quick results for its small datasets. For more information, see defining a benchmark.</p> </li> </ul> <p><code>Mode (optional, default='local')</code></p> <ul> <li> <p>The benchmark can be run in four modes:</p> <ul> <li><code>local</code>: install a local virtual environment and run the benchmark on your machine.</li> <li><code>docker</code>: create a docker image with the virtual environment and run the benchmark in a container on your machine.               If a local or remote image already exists, that will be used instead. Requires Docker.</li> <li><code>singularity</code>: create a singularity image with the virtual environment and run the benchmark in a container on your machine. Requires Singularity.</li> <li><code>aws</code>: run the benchmark on AWS EC2 instances.           It is possible to run directly on the instance or have the EC2 instance run in <code>docker</code> mode.           Requires valid AWS credentials to be configured, for more information see Running on AWS.</li> </ul> </li> </ul> <p>For a full list of parameters available, run:</p> <pre><code>python runbenchmark.py --help\n</code></pre>"},{"location":"benchmark/automl/specific_task_fold_example/","title":"Example: AutoML on a specific task and fold","text":"<p>The defaults are very useful for performing a quick test, as the datasets are small and cover different task types (binary classification, multiclass classification, and  regression). We also have a \"validation\" benchmark suite for more elaborate testing that also includes missing data, categorical data,  wide data, and more. The benchmark defines 9 tasks, and evaluating two folds with a 10-minute time constraint would take roughly 3 hours (=9 tasks * 2 folds * 10 minutes, plus overhead). Let's instead use the <code>--task</code> and <code>--fold</code> parameters to run only a specific task and fold in the <code>benchmark</code> when evaluating the  flaml AutoML framework:</p> <pre><code>python runbenchmark.py flaml validation test -t eucalyptus -f 0\n</code></pre> <p>This should take about 10 minutes plus the time it takes to install <code>flaml</code>. Results should look roughly like this:</p> <pre><code>Processing results for flaml.validation.test.local.20230711T122823\nSumming up scores for current run:\n               id       task  fold framework constraint    result      metric  duration       seed\nopenml.org/t/2079 eucalyptus     0     flaml       test -0.702976 neg_logloss     611.0 1385946458\n</code></pre> <p>Similarly to the test run, you will find additional files in the <code>results</code> directory.</p>"},{"location":"concepts/","title":"Concepts","text":""},{"location":"concepts/#openml-concepts","title":"OpenML concepts","text":"<p>OpenML operates on a number of core concepts which are important to understand: </p> <p> Datasets Datasets are pretty straight-forward. Tabular datasets are self-contained, consisting of a number of rows (instances) and columns (features), including their data types. Other  modalities (e.g. images) are included via paths to files stored within the same folder. Datasets are uniformly formatted (S3 buckets with Parquet tables, JSON metadata, and media files), and are auto-converted and auto-loaded in your desired format by the APIs (e.g. in Python) in a single line of code. Example: The Iris dataset or the Plankton dataset</p> <p> Tasks A task consists of a dataset, together with a machine learning task to perform, such as classification or clustering and an evaluation method. For supervised tasks, this also specifies the target column in the data. Example: Classifying different iris species from other attributes and evaluate using 10-fold cross-validation.</p> <p> Flows A flow identifies a particular machine learning algorithm (a pipeline or untrained model) from a particular library or framework, such as scikit-learn, pyTorch, or MLR. It contains details about the structure of the model/pipeline, dependencies (e.g. the library and its version) and a list of settable hyperparameters. In short, it is a serialized description of the algorithm that in many cases can also be deserialized to reinstantiate the exact same algorithm in a particular library.  Example: scikit-learn's RandomForest or a simple TensorFlow model</p> <p> Runs A run is an experiment - it evaluates a particular flow (pipeline/model) with particular hyperparameter settings, on a particular task. Depending on the task it will include certain results, such as model evaluations (e.g. accuracies), model predictions, and other output files (e.g. the trained model). Example: Classifying Gamma rays with scikit-learn's RandomForest</p>"},{"location":"concepts/authentication/","title":"Authentication","text":"<p>OpenML is as open as possible. You can download and inspect all datasets, tasks, flows and runs through the website or the API without creating an account.</p> <p>However, if you want to upload datasets or experiments, you need to create an account, sign in, and find your API key on your profile page. This key can then be used with any of the OpenML APIs.</p>"},{"location":"concepts/authentication/#api-keys","title":"API keys","text":"<p>If you don\u2019t have an account yet, sign up now. You will receive an API key, which will authenticate you to the server and allow you to download and upload datasets, tasks, runs and flows.</p> <ul> <li>Create an OpenML account (free) on https://www.openml.org.</li> <li>After logging in, open your profile page. Click on the avatar on the top right, and choose 'Your Profile'.</li> <li>Click on 'API key' to find your API key. You can also reset it if needed.</li> </ul> <p>To store your API key locally (to permanently authenticate), create a plain text file ~/.openml/config with the line 'apikey=MYKEY', replacing MYKEY with your API key. The config file must be in the directory ~/.openml/config and exist prior to importing the openml module.</p>"},{"location":"concepts/benchmarking/","title":"Collections and benchmarks","text":"<p>You can combine tasks and runs into collections, to run experiments across many tasks at once and collect all results. Each collection gets its own page, which can be linked to publications so that others can find all the details online.</p>"},{"location":"concepts/benchmarking/#benchmarking-suites","title":"Benchmarking suites","text":"<p>Collections of tasks can be published as benchmarking suites. Seamlessly integrated into the OpenML platform, benchmark suites standardize the setup, execution, analysis, and reporting of benchmarks. Moreover, they make benchmarking a whole lot easier: - all datasets are uniformly formatted in standardized data formats - they can be easily downloaded programmatically through APIs and client libraries - they come with machine-readable meta-information, such as the occurrence of missing  values, to train algorithms correctly - standardized train-test splits are provided to ensure that results can be objectively compared - results can be shared in a reproducible way through the APIs - results from other users can be easily downloaded and reused </p> <p>You can search for all existing benchmarking suites or create your own. For all further details, see the benchmarking guide.</p> <p></p>"},{"location":"concepts/benchmarking/#benchmark-studies","title":"Benchmark studies","text":"<p>Collections of runs can be published as benchmarking studies. They contain the results of all runs (possibly millions) executed on a specific benchmarking suite. OpenML allows you to easily download all such results at once via the APIs, but also visualized them online in the Analysis tab (next to the complete list of included tasks and runs). Below is an example of a benchmark study for AutoML algorithms.</p> <p></p>"},{"location":"concepts/data/","title":"Data","text":""},{"location":"concepts/data/#discovery","title":"Discovery","text":"<p>OpenML allows fine-grained search over thousands of machine learning datasets. Via the website, you can filter by many dataset properties, such as size, type, format, and many more. Via the APIs you have access to many more filters, and you can download a complete table with statistics of all datasest. Via the APIs you can also load datasets directly into your preferred data structures such as numpy (example in Python). We are also working on better organization of all datasets by topic </p> <p></p>"},{"location":"concepts/data/#sharing","title":"Sharing","text":"<p>You can upload and download datasets through the website or though the APIs (recommended). You can share data directly from common data science libraries, e.g. from Python or R dataframes, in a few lines of code. The OpenML APIs will automatically extract lots of meta-data and store all datasets in a uniform format.</p> <pre><code>    import pandas as pd\n    import openml as oml\n\n    # Create an OpenML dataset from a pandas dataframe\n    df = pd.DataFrame(data, columns=attribute_names)\n    my_data = oml.datasets.functions.create_dataset(\n        name=\"covertype\", description=\"Predicting forest cover ...\",\n        licence=\"CC0\", data=df\n    )\n\n    # Share the dataset on OpenML\n    my_data.publish()\n</code></pre> <p>Every dataset gets a dedicated page on OpenML with all known information, and can be edited further online.</p> <p></p> <p>Data hosted elsewhere can be referenced by URL. We are also working on interconnecting OpenML with other machine learning data set repositories </p>"},{"location":"concepts/data/#automated-analysis","title":"Automated analysis","text":"<p>OpenML will automatically analyze the data and compute a range of data quality characteristics. These include simple statistics such as the number of examples and features, but also potential quality issues (e.g. missing values) and more advanced statistics (e.g. the mutual information in the features and benchmark performances of simple models). These can be useful to find, filter and compare datasets, or to automate data preprocessing.  We are also working on simple metrics and automated dataset quality reports </p> <p>The Analysis tab (see image below, or try it live) also shows an automated and interactive analysis of all datasets. This runs on open-source Python code via Dash and we welcome all contributions </p> <p></p> <p>The third tab, 'Tasks', lists all tasks created on the dataset. More on that below.</p>"},{"location":"concepts/data/#dataset-id-and-versions","title":"Dataset ID and versions","text":"<p>A dataset can be uniquely identified by its dataset ID, which is shown on the website and returned by the API. It's <code>1596</code> in the <code>covertype</code> example above. They can also be referenced by name and ID. OpenML assigns incremental version numbers per upload with the same name. You can also add a free-form <code>version_label</code> with every upload.</p>"},{"location":"concepts/data/#dataset-status","title":"Dataset status","text":"<p>When you upload a dataset, it will be marked <code>in_preparation</code> until it is (automatically) verified. Once approved, the dataset will become <code>active</code> (or <code>verified</code>). If a severe issue has been found with a dataset, it can become <code>deactivated</code> (or <code>deprecated</code>) signaling that it should not be used. By default, dataset search only returns verified datasets, but you can access and download datasets with any status.</p>"},{"location":"concepts/data/#special-attributes","title":"Special attributes","text":"<p>Machine learning datasets often have special attributes that require special handling in order to build useful models. OpenML marks these as special attributes.</p> <p>A <code>target</code> attribute is the column that is to be predicted, also known as dependent variable. Datasets can have a default target attribute set by the author, but OpenML tasks can also overrule this. Example: The default target variable for the MNIST dataset is to predict the class from pixel values, and most supervised tasks will have the class as their target. However, one can also create a task aimed at predicting the value of pixel257 given all the other pixel values and the class column.</p> <p><code>Row id</code> attributes indicate externally defined row IDs (e.g. <code>instance</code> in dataset 164). <code>Ignore</code> attributes are other columns that should not be included in training data (e.g. <code>Player</code> in dataset 185). OpenML will clearly mark these, and will (by default) drop these columns when constructing training sets.</p>"},{"location":"concepts/flows/","title":"Flows","text":"<p>Flows are machine learning pipelines, models, or scripts. They are typically uploaded directly from machine learning libraries (e.g. scikit-learn, pyTorch, TensorFlow, MLR, WEKA,...) via the corresponding APIs. Associated code (e.g., on GitHub) can be referenced by URL.</p>"},{"location":"concepts/flows/#analysing-algorithm-performance","title":"Analysing algorithm performance","text":"<p>Every flow gets a dedicated page with all known information. The Analysis tab shows an automated interactive analysis of all collected results. For instance, below are the results of a scikit-learn pipeline including missing value imputation, feature encoding, and a RandomForest model. It shows the results across multiple tasks, and how the AUC score is affected by certain hyperparameters.</p> <p></p> <p>This helps to better understand specific models, as well as their strengths and weaknesses.</p>"},{"location":"concepts/flows/#automated-sharing","title":"Automated sharing","text":"<p>When you evaluate algorithms and share the results, OpenML will automatically extract all the details of the algorithm (dependencies, structure, and all hyperparameters), and upload them in the background.</p> <pre><code>    from sklearn import ensemble\n    from openml import tasks, runs\n\n    # Build any model you like.\n    clf = ensemble.RandomForestClassifier()\n\n    # Evaluate the model on a task\n    run = runs.run_model_on_task(clf, task)\n\n    # Share the results, including the flow and all its details.\n    run.publish()\n</code></pre>"},{"location":"concepts/flows/#reproducing-algorithms-and-experiments","title":"Reproducing algorithms and experiments","text":"<p>Given an OpenML run, the exact same algorithm or model, with exactly the same hyperparameters, can be reconstructed within the same machine learning library to easily reproduce earlier results. </p> <pre><code>    from openml import runs\n\n    # Rebuild the (scikit-learn) pipeline from run 9864498\n    model = openml.runs.initialize_model_from_run(9864498)\n</code></pre> <p>Note</p> <p>You may need the exact same library version to reconstruct flows. The API will always state the required version. We aim to add support for VMs so that flows can be easily (re)run in any environment </p>"},{"location":"concepts/runs/","title":"Runs","text":""},{"location":"concepts/runs/#automated-reproducible-evaluations","title":"Automated reproducible evaluations","text":"<p>Runs are experiments (benchmarks) evaluating a specific flows on a specific task. As shown above, they are typically submitted automatically by machine learning libraries through the OpenML APIs), including lots of automatically extracted meta-data, to create reproducible experiments. With a few for-loops you can easily run (and share) millions of experiments.</p>"},{"location":"concepts/runs/#online-organization","title":"Online organization","text":"<p>OpenML organizes all runs online, linked to the underlying data, flows, parameter settings, people, and other details. See the many examples above, where every dot in the scatterplots is a single OpenML run.</p>"},{"location":"concepts/runs/#independent-server-side-evaluation","title":"Independent (server-side) evaluation","text":"<p>OpenML runs include all information needed to independently evaluate models. For most tasks, this includes all predictions, for all train-test splits, for all instances in the dataset, including all class confidences. When a run is uploaded, OpenML automatically evaluates every run using a wide array of evaluation metrics. This makes them directly comparable with all other runs shared on OpenML. For completeness, OpenML will also upload locally computed evaluation metrics and runtimes. </p> <p>New metrics can also be added to OpenML's evaluation engine, and computed for all runs afterwards. Or, you can download OpenML runs and analyse the results any way you like.</p> <p>Note</p> <p>Please note that while OpenML tries to maximise reproducibility, exactly reproducing all results may not always be possible because of changes in numeric libraries,  operating systems, and hardware.</p>"},{"location":"concepts/sharing/","title":"Sharing (under construction)","text":"<p>Currently, anything on OpenML can be shared publicly or kept private to a single user. We are working on sharing features that allow you to share your materials with other users without making them entirely public. Watch this space </p>"},{"location":"concepts/tagging/","title":"Tagging","text":"<p>Datasets, tasks, runs and flows can be assigned tags, either via the web interface or the API. These tags can be used to search and annotate datasets, or simply to better organize your own datasets and experiments.</p> <p>For example, the tag OpenML-CC18 refers to all tasks included in the OpenML-CC18 benchmarkign suite.</p>"},{"location":"concepts/tasks/","title":"Tasks","text":"<p>Tasks describe what to do with the data. OpenML covers several task types, such as classification and clustering. Tasks are containers including the data and other information such as train/test splits, and define what needs to be returned. They are machine-readable so that you can automate machine learning experiments, and easily compare algorithms evaluations (using the exact same train-test splits) against all other benchmarks shared by others on OpenML.</p>"},{"location":"concepts/tasks/#collaborative-benchmarks","title":"Collaborative benchmarks","text":"<p>Tasks are real-time, collaborative benchmarks (e.g. see MNIST below). In the Analysis tab, you can view timelines and leaderboards, and learn from all prior submissions to design even better algorithms.</p> <p></p>"},{"location":"concepts/tasks/#discover-the-best-algorithms","title":"Discover the best algorithms","text":"<p>All algorithms evaluated on the same task (with the same train-test splits) can be directly compared to each other, so you can easily look up which algorithms perform best overall, and download their exact configurations. Likewise, you can look up the best algorithms for similar tasks to know what to try first.</p> <p></p>"},{"location":"concepts/tasks/#automating-benchmarks","title":"Automating benchmarks","text":"<p>You can search and download existing tasks, evaluate your algorithms, and automatically share the results (which are stored in a run). Here's what this looks like in the Python API. You can do the same across hundreds of tasks at once.</p> <pre><code>    from sklearn import ensemble\n    from openml import tasks, runs\n\n    # Build any model you like\n    clf = ensemble.RandomForestClassifier()\n\n    # Download any OpenML task (includes the datasets)\n    task = tasks.get_task(3954)\n\n    # Automatically evaluate your model on the task\n    run = runs.run_model_on_task(clf, task)\n\n    # Share the results on OpenML.\n    run.publish()\n</code></pre> <p>You can create new tasks via the website or via the APIs as well.</p>"},{"location":"contributing/","title":"How to Contribute","text":"<p>OpenML is an open source project, hosted on GitHub. We welcome everybody to help improve OpenML, and make it more useful for everyone.</p> <p>Mission</p> <p>We want to make machine learning open and accessible for the benefit of all of humanity. OpenML offers an entirely open online platform for machine learning datasets, models, and experiments, making them easy to use and share to facilitate global collaboration and extensive automation.</p>"},{"location":"contributing/#want-to-get-involved","title":"Want to get involved?","text":"<p>Awesome, we're happy to have you! </p>"},{"location":"contributing/#who-are-we","title":"Who are we?","text":"<p>We are a group of friendly people who are excited about open science and machine learning. </p> <p>Read more about who we are, what we stand for, and how to get in touch.</p>"},{"location":"contributing/#we-need-help","title":"We need help!","text":"<p>We are currently looking for help with:</p> <p> User feedback (best via GitHub issues, but email or Slack is also fine)</p> <ul> <li>Frontend / UX / Design of the website</li> <li>Backend / API</li> <li>Outreach / making OpenML better known (especially in non-ML-communities, where people have data but no analysis experise)</li> <li>Helping with the interfaces (Python,R,Julia,Java) and tool integrations</li> <li>Helping with documenting the interfaces or the API</li> <li>What could we do better to get new users started? Help us to figure out what is difficult to understand about OpenML. If you are a new user, you are the perfect person for this!</li> </ul>"},{"location":"contributing/#beginner-issues","title":"Beginner issues","text":"<p>Check out the issues labeled Good first issue or help wanted (you need to be logged into GitHub to see these)</p>"},{"location":"contributing/#change-the-world","title":"Change the world","text":"<p>If you have your own ideas on how you want to contribute, please get in touch! We are very friendly and open to new ideas </p>"},{"location":"contributing/#communication-channels","title":"Communication channels:","text":"<p>We have several communication channels set up for different purposes:</p>"},{"location":"contributing/#github","title":"GitHub","text":"<p>https://github.com/openml</p> <ul> <li>Issues (members and users can complain)</li> <li>Request new features</li> </ul> <p>Anyone with a GitHub account can write issues. We are happy if people get involved by writing issues, so don't be shy </p> <p>Please post issues in the relevant issue tracker.</p> <ul> <li> OpenML Core - Web services and API</li> <li> Website - The (new) OpenML website</li> <li> Docs - The documentation pages</li> <li> Python API - The Python API</li> <li> R API - The OpenML R package</li> <li> Java API - The Java API and Java-based plugins</li> <li> Datasets - For issues about datasets</li> <li> Blog - The OpenML Blog</li> </ul>"},{"location":"contributing/#slack","title":"Slack","text":"<p>https://openml.slack.com</p> <ul> <li>Informal communication</li> </ul> <p>We use slack for day to day discussions and news. If you want to join the OpenML slack chat, please message us (openmlHQ@googlegroups.com).</p>"},{"location":"contributing/#twitter-open_ml","title":"Twitter (@open_ml)","text":"<p>https://twitter.com/open_ml</p> <ul> <li>News</li> <li>Publicly relevant information</li> </ul>"},{"location":"contributing/#blog","title":"Blog","text":"<p>https://blog.openml.org</p> <ul> <li>Tutorials</li> <li>News</li> <li>Open discussions</li> </ul>"},{"location":"contributing/#contributors-bot","title":"Contributors bot","text":"<p>We use all contributors bot to add contributors to the repository README. You can check how to use this here. You can contribute in a lot of ways including code, blogs, content, design and talks. You can find the emoji key here .</p>"},{"location":"contributing/OpenML-Docs/","title":"Documentation","text":""},{"location":"contributing/OpenML-Docs/#general-documentation","title":"General Documentation","text":"<p>High-quality and up-to-date documentation are crucial. If you notice any mistake in these documentation pages, click the  button (on the top right). It will open up an editing page on GitHub (you do need to be logged in). When you are done, add a small message explaining the change and click 'commit changes'. On the next page, just launch the pull request. We will then review it and approve the changes, or discuss them if necessary.</p> <p>The sources are generated by MkDocs, using the Material theme. Check these docs to see what is possible in terms of styling.</p> <p>OpenML is a big project with multiple repositories. To keep the documentation close to the code, it will always be kept in the relevant repositories (see below), and  combined into these documentation pages using MkDocs multirepo.</p> <p>Developer note</p> <p>To work on the documentation locally, do the following: <pre><code>git clone https://github.com/openml/docs.git\npip install -r requirements.txt\n</code></pre> To build the documentation, run <code>mkdocs serve</code> in the top directory (with the <code>mkdocs.yml</code> file). Any changes made after that will be hot-loaded.</p> <p>The documentation will be auto-deployed with every push or merge with the master branch of <code>https://www.github.com/openml/docs/</code>. In the background, a CI job will run <code>mkdocs gh-deploy</code>, which will build the HTML files and push them to the gh-pages branch of openml/docs. <code>https://docs.openml.org</code> is just a reverse proxy for <code>https://openml.github.io/docs/</code>.</p>"},{"location":"contributing/OpenML-Docs/#python-api","title":"Python API","text":"<p>To edit the tutorial, you have to edit the <code>reStructuredText</code> files on openml-python/doc. When done, you can do a pull request.</p> <p>To edit the documentation of the python functions, edit the docstrings in the Python code. When done, you can do a pull request.</p> <p>Developer note</p> <p>A CircleCI job will automatically render the documentation on every GitHub commit, using Sphinx. For inclusion in these documentation pages, it will also be rendered in markdown and imported.</p>"},{"location":"contributing/OpenML-Docs/#r-api","title":"R API","text":"<p>To edit the tutorial, you have to edit the <code>Rmarkdown</code> files on openml-r/vignettes.</p> <p>To edit the documentation of the R functions, edit the Roxygen documention next to the functions in the R code.</p> <p>Developer note</p> <p>A Travis job will automatically render the documentation on every GitHub commit, using knitr. The Roxygen documentation is updated every time a new version is released on CRAN.</p>"},{"location":"contributing/OpenML-Docs/#java-api","title":"Java API","text":"<p>The Java Tutorial is written in markdown and can be edited the usual way (see above).</p> <p>To edit the documentation of the Java functions, edit the documentation next to the functions in the Java code.</p> <ul> <li>Javadocs: https://www.openml.org/docs/</li> </ul> <p>Developer note</p> <p>A Travis job will automatically render the documentation on every GitHub commit, using Javadoc.</p>"},{"location":"contributing/OpenML-Docs/#rest-api","title":"REST API","text":"<p>The REST API is documented using Swagger.io, in YAML. This generates a nice web interface that also allows trying out the API calls using your own API key (when you are logged in).</p> <p>You can edit the sources on SwaggerHub. When you are done, export to json and replace the downloads/swagger.json file in the OpenML main GitHub repository. You need to do a pull request that is then reviewed by us. When we merge the new file the changes are immediately available.</p> <p>The data API can be edited in the same way.</p>"},{"location":"contributing/Style/","title":"Style guide","text":"<p>These are some (non-mandatory) style guidelines to make the OpenML experience more pleasant and consistent for everyone.</p>"},{"location":"contributing/Style/#logos","title":"Logos","text":"<p> (SVG)</p>"},{"location":"contributing/Style/#colors","title":"Colors","text":"<p>We use the Material Design color system, and especially the colors green[400], yellow[800], blue[800], red[400], green[400], yellow[800], pink[400], and purple[400].</p> <p>Primary colors are #1E88E5 (general), #000482 (dark), and #b5b7ff (light).</p>"},{"location":"contributing/resources/","title":"Resources","text":""},{"location":"contributing/resources/#resources","title":"Resources","text":""},{"location":"contributing/resources/#database-snapshots","title":"Database snapshots","text":"<p>Everything uploaded to OpenML is available to the community. The nightly snapshot of the public database contains all experiment runs, evaluations and links to datasets, implementations and result files. In SQL format (gzipped). You can also download the Database schema.</p> <p> Nightly database SNAPSHOT</p> <p>If you want to work on the website locally, you'll also need the schema for the 'private' database with non-public information.</p> <p> Private database schema</p>"},{"location":"contributing/resources/#legacy-resources","title":"Legacy Resources","text":"<p>OpenML is always evolving, but we keep hosting the resources that were used in prior publications so that others may still build on them.</p> <p> The experiment database used in Vanschoren et al. (2012) Experiment databases. Machine Learning 87(2), pp 127-158. You'll need to import this database (we used MySQL) to run queries. The database structure is described in the paper. Note that most of the experiments in this database have been rerun using OpenML, using newer algorithm implementations and stored in much more detail.</p> <p> The Expos\u00e9 ontology used in the same paper, and described in more detail here and here. Expos\u00e9 is used in designing our databases, and we aim to use it to export all OpenML data as Linked Open Data.</p>"},{"location":"contributing/resources/#other-dataset-repositories","title":"Other dataset repositories","text":"<p>We keep a list of other dataset repositories all over the world</p>"},{"location":"contributing/backend/API-development/","title":"Code structure","text":"<p>Phasing out</p> <p>This documentation is about the old PHP-based API, which wil be phased out in favor of (e.g. the new Python-based API (using FastAPI)). See the 'Server' tab for more information.</p>"},{"location":"contributing/backend/API-development/#important-resources","title":"Important resources","text":"<p>REST API docs: www.openml.org/apis</p> <p>Controller: https://github.com/openml/OpenML/blob/master/openml_OS/controllers/Api_new.php</p> <p>Models: https://github.com/openml/OpenML/tree/master/openml_OS/models/api/v1</p> <p>Templates: https://github.com/openml/OpenML/tree/master/openml_OS/views/pages/api_new/v1</p>"},{"location":"contributing/backend/API-development/#golden-rules-for-development","title":"Golden Rules for Development","text":"<ol> <li>Code Maintainability before anything else. The code has to be understandable, and if not conflicting with that, short. Avoid code duplications as much as possible.</li> <li>The API controller is the only entity giving access to the API models. Therefore, the responsibility for API access can be handled by the controller</li> <li>Read-Only operations are of the type GET. Operations that make changes in the database are of type POST or DELETE. Important, because this is the way the controller determines to allow users with a given set of privileges to access functions.</li> <li>Try to avoid direct queries to the database. Instead, use the respective models functions: 'get()', 'getWhere()', 'getById()', insert(), etc (Please make yourself familiar with the basic model: read-only and write)</li> <li>No external program/script execution during API calls (with one exception: data split generation). This makes the API unnecessarily slow, hard to debug and vulnerable to crashes. If necessary, make a cronjob that executes the program / script</li> </ol>"},{"location":"contributing/backend/API-development/#backend-code-structure","title":"Backend code structure","text":"<p>The high-level architecture of the website, including the controllers for different parts of the website (REST API, html, ...) and connections to the database.</p>"},{"location":"contributing/backend/API-development/#code","title":"Code","text":"<p>The source code is available in the 'OpenML' repository: https://github.com/openml/OpenML </p>"},{"location":"contributing/backend/API-development/#important-files-and-folders","title":"Important files and folders","text":"<p>In this section we go through all important files and folder of the system.</p>"},{"location":"contributing/backend/API-development/#root-directory","title":"Root directory","text":"<p>The root directory of OpenML contains the following files and folders.</p> <ul> <li> <p>system: This folder contains all files provided by   CodeIgniter 2.1.3. The contents of this folder is   beyond the scope of this document, and not relevant for extending   OpenML. All the files in this folder are in the same state as they   were provided by Ellislabs, and none of these files should ever be   changed.</p> </li> <li> <p>sparks: Sparks is a package management system for   Codeigniter that allows for instant installation of libraries into   the application. This folder contains two libraries provided by   third party software developers, oauth1 (based on version 1 the   oauth protocol) and oauth2 (similarly, based on version 2 of the   oauth protocol). The exact contents of this folder is beyond the   scope of this document and not relevant for extending OpenML.</p> </li> <li> <p>openml_OS: All files in this folder are written specifically   for OpenML. When extending the functionality OpenML, usually one of   the files in this folder needs to be adjusted. As a thorough   understanding of the contents of this folder is vital for extending   OpenML, we will discuss the contents of this folder in   [[URL Mapping]] in more detail.</p> </li> <li> <p>index.php: This is the \u201cbootstrap\u201d file of the system.   Basically, every page request on OpenML goes through this file (with   the css, images and javascript files as only exception). It then   determines which CodeIgniter and OpenML files need to be included.   This file should not be edited.</p> </li> <li> <p>.htaccess: This file (which configures the Apache Rewrite   Engine) makes sure that all URL requests will be directed to   <code>index.php</code>. Without this file, we would need to include <code>index.php</code>   explicitly in every URL request. This file makes sure that all other   URL requests without <code>index.php</code> embedded in it automatically will   be transformed to <code>index.php</code>. Eg.,   http://www.openml.org/frontend/page/home will be rewritten to   http://www.openml.org/index.php/frontend/page/home. This will be   explained in detail in [[URL Mapping]].</p> </li> <li> <p>css: A folder containing all stylesheets. These are important   for the layout of OpenML.</p> </li> <li> <p>data: A folder containing data files, e.g., datasets,   implementation files, uploaded content. Please note that this folder   does not necessarily needs to be present in the root directory. The   OpenML Base Config file determines the   exact location of this folder.</p> </li> <li> <p>downloads: Another data folder, containing files like the most   recent database snapshot.</p> </li> <li> <p>img: A folder containing all static images shown on the webpage.</p> </li> <li> <p>js: A folder containing all used Javascript files and libraries,   including third party libraries like jQuery and datatables.</p> </li> <li> <p>Various other files, like .gitignore, favicon.ico, etc.</p> </li> </ul>"},{"location":"contributing/backend/API-development/#openml_os","title":"openml_OS","text":"<p>This folder is (in CodeIgniter jargon) the \u201cApplication folder\u201d, and contains all files relevant to OpenML. Within this folder, the following folders should be present: (And also some other folders, but these are not used by OpenML)</p> <ul> <li> <p>config: A folder containing all config files. Most notably, it   contains the file BASE_CONFIG.php, in which all system   specific variables are set; the config items within this file   differs over various installations (e.g., on localhost,   <code>openml.org</code>). Most other config files, like   database.php, will receive their values from   BASE_CONFIG.php. Other important config files are   autoload.php, determining which CodeIgniter / OpenML   files will be loaded on any request, openML.php,   containing config items specific to OpenML, and   routes.php, which will be explained in   [[URL Mapping]].</p> </li> <li> <p>controllers: In the Model/View/Controller design pattern, all   user interaction goes through controllers. In a webapplication   setting this means that every time a URL gets requested, exactly one   controller gets invoked. The exact dynamics of this will be   explained in [[URL Mapping]].</p> </li> <li> <p>core: A folder that contains CodeIgniter specific files. These   are not relevant for the understanding of OpenML.</p> </li> <li> <p>helpers: This folder contains many convenience functions.   Wikipedia states: \u201cA convenience function is a non-essential   subroutine in a programming library or framework which is intended   to ease commonly performed tasks\u201d. For example the   file_upload_helper.php contains many functions that   assist with uploading of files. Please note that a helper function   must be explicitly loaded in either the autoload config or the files   that uses its functions.</p> </li> <li> <p>libraries: Similar to sparks, this folder contains libraries   specifically written for CodeIgniter. For example, the library used   for all user management routines is in this folder.</p> </li> <li> <p>models: In the Model/View/Controller design pattern, models   represent the state of the system. In a webapplication setting, you   could say that a model is the link to the database. In OpenML,   almost all tables of the database are represented by a model. Each   model has general functionality applicable to all models (e.g.,   retrieve all records, retrieve record with constraints, insert   record) and functionality specific to that model (e.g., retrieve a   dataset that has certain data properties). Most models extend an   (abstract) base class, located in the abstract folder.   This way, all general functionality is programmed and maintained in   one place.</p> </li> <li> <p>third_party: Although the name might suggests differently, this   folder contains all OpenML Java libraries.</p> </li> <li> <p>views: In the Model/View/Controller design pattern, the views   are the way information is presented on the screen. In a   webapplication setting, a view usually is a block of (PHP generated)   HTML code. The most notable view is frontend_main.php,   which is the template file determining the main look and feel of   OpenML. Every single page also has its own specific view (which is   parsed within frontend_main.php). These pages can be   found (categorized by controller and name) in the pages   folder. More about this structure is explained in   [[URL Mapping]].</p> </li> </ul>"},{"location":"contributing/backend/API-development/#frontend-code-structure","title":"Frontend code structure","text":"<p>Architecture and libraries involved in generating the frontend functions.</p> <p>Code: https://github.com/openml/website/tree/master/openml_OS/views</p>"},{"location":"contributing/backend/API-development/#high-level","title":"High-level","text":"<p>All pages are generated by first loading frontend_main.php. This creates the 'shell' in which the content is loaded. It loads all css and javascript libraries, and contains the html for displaying headers and footers.</p>"},{"location":"contributing/backend/API-development/#create-new-page","title":"Create new page","text":"<p>The preferred method is creating a new folder into the folder <code>&lt;root_directory&gt;/openml_OS/views/pages/frontend</code> This page can be requested by <code>http://www.openml.org/frontend/page/&lt;folder_name&gt;</code> or just <code>http://www.openml.org/&lt;folder_name&gt;</code> This method is preferred for human readable webpages, where the internal actions are simple, and the output is complex. We will describe the files that can be in this folder.</p> <ul> <li> <p>pre.php: Mandatory file. Will be executed first. Do not make   this file produce any output! Can be used to pre-render data, or set   some variables that are used in other files.</p> </li> <li> <p>body.php: Highly recommended file. Intended for displaying the   main content of this file. Will be rendered at the right location   within the template file (<code>frontend_main.php</code>).</p> </li> <li> <p>javascript.php: Non-mandatory file. Intended for javascript   function on which <code>body.php</code> relies. Will be rendered within a   javascript block in the header of the page.</p> </li> <li> <p>post.php: Non mandatory file. Will only be executed when a POST   request is done (e.g., when a HTML form was send using the POST   protocol). Will be executed after <code>pre.php</code>, but before the   rendering process (and thus, before <code>body.php</code> and   <code>javascript.php</code>). Should handle the posted input, e.g., file   uploads.</p> </li> </ul> <p>It is also recommended to add the newly created folder to the mapping in the <code>routes.php</code> config file. This way it can also be requested by the shortened version of the URL. (Note that we deliberately avoided to auto-load all pages into this file using a directory scan, as this makes the webplatform slow.)</p>"},{"location":"contributing/backend/API-development/#url-to-page-mapping","title":"URL to Page Mapping","text":"<p>Most pages in OpenML are represented by a folder in /openml_OS/views/pages/frontend The contents of this folder will be parsed in the template <code>frontend_main.php</code> template, as described in [[backend]]. In this section we explain the way an URL is mapped to a certain OpenML page."},{"location":"contributing/backend/API-development/#url-anatomy","title":"URL Anatomy","text":"<p>By default, CodeIgniter (and OpenML) accepts a URL in the following form: <code>http://www.openml.org/index.php/&lt;controller&gt;/&lt;function&gt;/&lt;p1&gt;/&lt;pN&gt;/&lt;free&gt;</code> The various parts in the URL are divided by slashes. Every URL starts with the protocol and server name (in the case of OpenML this is <code>http://www.openml.org/</code>). This is followed by the bootstrap file, which is always the same, i.e., <code>index.php</code>. The next part indicates the controller that needs to be invoked; typically this is <code>frontend</code>, <code>rest_api</code> or <code>data</code>, but it can be any file from the <code>openml_OS</code> folder <code>controllers</code>. Note that the suffix <code>.php</code> should not be included in the URL.</p> <p>The next part indicates which function of the controller should be invoked. This should be a existing, public function from the controller that is indicated in the controller part. These functions might have one or more parameters that need to be set. This is the following part of the URL (indicated by <code>p1</code> and <code>pN</code>). The parameters can be followed by anything in free format. Typically, this free format is used to pass on additional parameters in <code>name</code> - <code>value</code> format, or just a way of adding a human readable string to the URL for SEO purposes.</p> <p>For example, the following URL <code>http://www.openml.org/index.php/frontend/page/home</code> invokes the function <code>page</code> from the <code>frontend</code> controller and sets the only parameter of this function, <code>$indicator</code>, to value <code>home</code>. The function <code>page</code> loads the content of the specified folder (<code>$indicator</code>) into the main template. In this sense, the function <code>page</code> can be seen as some sort of specialized page loader.</p>"},{"location":"contributing/backend/API-development/#url-shortening","title":"URL Shortening","text":"<p>Since it is good practice to have URL\u2019s as short as possible, we have introduced some logic that shortens the URL\u2019s. Most importantly, the URL part that invokes <code>index.php</code> can be removed at no cost, since this file is always invoked. For this, we use Apache\u2019s rewrite engine. Rules for rewriting URL\u2019s can be found in the <code>.htaccess</code> file, but is suffices to say that any URL in the following format <code>http://www.openml.org/index.php/&lt;controller&gt;/&lt;function&gt;/&lt;params&gt;</code> can due to the rewrite engine also be requested with <code>http://www.openml.org/&lt;controller&gt;/&lt;function&gt;/&lt;params&gt;</code></p> <p>Furthermore, since most of the pages are invoked by the function <code>page</code> of the <code>frontend</code> controller (hence, they come with the suffix <code>frontend/page/page_name</code>) we also created a mapping that maps URL\u2019s in the following form <code>http://www.openml.org/&lt;page_name&gt;</code> to <code>http://www.openml.org/frontend/page/&lt;page_name&gt;</code> Note that Apache\u2019s rewrite engine will also add <code>index.php</code> to this. The exact mapping can be found in <code>routes.php</code> config file.</p>"},{"location":"contributing/backend/API-development/#additional-mappings","title":"Additional Mappings","text":"<p>Additionally, a mapping is created from the following type of URL: <code>http://www.openml.org/api/&lt;any_query_string&gt;</code> to <code>http://www.openml.org/rest_api/&lt;any_query_string&gt;</code> This was done for backwards compatibility. Many plugins make calls to the not-existing <code>api</code> controller, which are automatically redirected to the <code>rest_api</code> controller.</p>"},{"location":"contributing/backend/API-development/#exceptions","title":"Exceptions","text":"<p>It is important to note that not all pages do have a specific page folder. The page folders are a good way of structuring complex GUI\u2019s that need to be presented to the user, but in cases where the internal state changes are more important than the GUI\u2019s, it might be preferable to make the controller function print the output directly. This happens for example in the functions of <code>rest_api.php</code> and <code>free_query.php</code> (although the former still has some files in the views folder that it refers to).</p>"},{"location":"contributing/backend/API-development/#xsd-schemas","title":"XSD Schemas","text":"<p>In order to ensure data integrity on the server, data that passed to upload functions is checked against XSD schema's. This ensures that the data that is uploaded is in the correct format, and does not contain any illegal characters. XSD schema's can be obtained through the API (exact links are provided in the API docs, but for example: https://www.openml.org/api/v1/xsd/openml.data.upload (where openml.data.upload can be replaced by any other schema's name). Also XML examples are provided, e.g., https://www.openml.org/api/v1/xml_example/data . The XSD schema's are exactly the same as used on the server. Whenever an upload fails and the server mentions an XML/XSD verification error, please run the uploaded xml against one of the provided XSD schema's, for example on this webtool: http://www.freeformatter.com/xml-validator-xsd.html</p> <p>In order to maintain one XSD schema for both uploading and downloading stuff, the XSD sometimes contains more fields than seem necessary from the offset. Usually, the additional fields that are indicated as such in the comments (for example, in the upload dataset xsd this are the id, upload_date, etc fields). The XSD's maintain basically three consistencies functions:</p> <ul> <li>Ensure that the correct fields are uploaded</li> <li>Ensure that the fields contain the correct data types.</li> <li>Ensure that the fields do not contain to much characters for the database to upload.</li> </ul> <p>For the latter two, it is important to note that the XSD seldom accept default string content (i.e., xs:string). Rather, we use self defined data types, that use regular expressions to ensure the right content. Examples of these are oml:system_string128, oml:casual_string128, oml:basic_latin128, where the oml prefix is used, the name indicates the level of restriction and the number indicates the maximum size of the field.</p> <p>IMPORTANT: The maximum field sizes are (often) chosen with great care. Do not extend them without consulting other team members.</p>"},{"location":"contributing/backend/API-development/#user-authentication","title":"User authentication","text":"<p>Authentication towards the server goes by means of a so-called api_key (a hexa-decimal string which uniquely identifies a user). Upon interaction with the server, the client passes this api_key to the server, and the server checks the rights of the user. Currently this goes by means of a get or post variable, but in the future we might want to use a header field (because of security). It is recommended to refresh your api_key every month.</p> <p>IMPORTANT: Most authentication operations are handled by the ION_Auth library (http://benedmunds.com/ion_auth/). DO NOT alter information directly in the user table, always use the ION_Auth API.</p> <p>A user can be part of one or many groups. The following user groups exists:</p> <ol> <li>Admin Group: With great power comes great responsibility. Admin users can overrule all security checks on the server, i.e., delete a dataset or run that is not theirs, or even delete a flow that contains runs.</li> <li>Normal Group: Level that is required for read/write interaction with the server. Almost all users are part of this group.</li> <li>Read-only Group: Level that can be used for read interaction with the server. If a user is part of this group, but not part of 'Normal Group', he is allowed to download content, but can not upload or delete content.</li> <li>Backend Group: (Work in Progress) Level that has more privileges than 'Normal Group'. Can submit Data Qualities and Evaluations.</li> </ol> <p>The ION_Auth functions in_group(), add_to_group(), remove_from_group() and get_users_groups() are key towards interaction with these tables.</p>"},{"location":"contributing/backend/Datasets/","title":"Datasets","text":""},{"location":"contributing/backend/Datasets/#data-formats","title":"Data Formats","text":"<p>OpenML aims to achieve full data interoperability, meaning that you can load all datasets in a uniform way (a 'universal dataloader'). This requires that all datasets are stored in the same dataformat (or a set of interoperable formats), or at least have a version of it stored in that format. After an intensive study, which you can read on our blog, we settled on the Parquet format.</p> <p>This means that all OpenML datasets can be retrieved in the Parquet format. They are also stored on our servers in this format. Oftentimes, you will not notice this, as the OpenML clients can automatically convert data into your preferred data structures, and be fed directly into machine learning workflows. For example:</p> <pre><code>import openml\ndataset = openml.datasets.get_dataset(\"Fashion-MNIST\")     # Returns the dataset meta-data \nX, y, _, _ = dataset.get_data(dataset_format=\"dataframe\",  # Downloads the data and returns a Pandas dataframe\n                target=dataset.default_target_attribute)\n\nfrom sklearn.ensemble import GradientBoostingClassifier         # Using a sklearn model as an example\nmodel = GradientBoostingClassifier(n_estimators=10).fit(X, y)   # Set hyperparameters and train the model \n</code></pre> <p>To guarantee interoperability, we focus on a limited set of data formats.</p>"},{"location":"contributing/backend/Datasets/#tabular-data","title":"Tabular data","text":"<p>OpenML has historically focussed on tabular data, and has extensive support for all kinds of tabular data. As explained above, we store all data in the Parquet format. You can upload data from many different data structures, such as Pandas dataframes and R dataframes, after which they will be converted and stored in Parquet. You can also upload datasets as CSV files or ARFF files, and we aim to allow direct Parquet uploads soon.</p> <p>ARFF legacy</p> <p>At the moment, some aspects of OpenML still has a dependency on the ARFF format. This will be fully phased out in favor of Parquet.</p>"},{"location":"contributing/backend/Datasets/#image-data","title":"Image data","text":"<p>OpenML generally supports other data types by requiring a 'header table', a table (stored in Parquet) listing all data instances with additional meta-data (e.g. classes, bounding boxes,...) and references to data files, such as images (e.g. JPGs), stored in seperate folders. See our blog post for details. We will provide more detailed guidelines here as soon as possible.</p>"},{"location":"contributing/backend/Datasets/#data-repositories","title":"Data repositories","text":"<p>This is a list of public dataset repositories that offer additional useful machine learning datasets. These have widely varying data formats, so they require manual selection, parsing and meta-data extraction.</p> <p>A collection of sources made by different users</p> <ul> <li>https://github.com/caesar0301/awesome-public-datasets</li> <li>https://dreamtolearn.com/ryan/1001_datasets</li> <li>https://en.wikipedia.org/wiki/List_of_datasets_for_machine-learning_research</li> <li>https://pathmind.com/wiki/open-datasets</li> <li>https://paperswithcode.com/</li> <li>https://medium.com/towards-artificial-intelligence/best-datasets-for-machine-learning-data-science-computer-vision-nlp-ai-c9541058cf4f</li> <li>https://lionbridge.ai/datasets/the-50-best-free-datasets-for-machine-learning/</li> <li>https://www.v7labs.com/open-datasets?utm_source=v7&amp;utm_medium=email&amp;utm_campaign=edu_outreach</li> </ul> <p>Machine learning dataset repositories (mostly already in OpenML)</p> <ul> <li>UCI: https://archive.ics.uci.edu/ml/index.html</li> <li>KEEL: http://sci2s.ugr.es/keel/datasets.php</li> <li>LIBSVM: http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/</li> <li>AutoWEKA datasets: http://www.cs.ubc.ca/labs/beta/Projects/autoweka/datasets/</li> <li>skData package: https://github.com/jaberg/skdata/tree/master/skdata</li> <li>Rdatasets: http://vincentarelbundock.github.io/Rdatasets/datasets.html</li> <li>DataBrewer: https://github.com/rmax/databrewer</li> <li>liac-arff: https://github.com/renatopp/arff-datasets</li> </ul> <p>MS Open datasets:</p> <ul> <li>https://azure.microsoft.com/en-us/services/open-datasets/catalog/</li> </ul> <p>APIs (mostly defunct):</p> <ul> <li>databrewer (Python): https://pypi.org/project/databrewer/</li> <li>PyDataset (Python): https://github.com/iamaziz/PyDataset (wrapper for Rdatasets?)</li> <li>RDatasets (R): https://github.com/vincentarelbundock/Rdatasets</li> </ul> <p>Time series / Geo data:</p> <ul> <li>Data commons: https://datacommons.org/</li> <li>UCR: http://timeseriesclassification.com/</li> <li>Older version: http://www.cs.ucr.edu/~eamonn/time_series_data/</li> </ul> <p>Deep learning datasets (mostly image data)</p> <ul> <li>https://www.tensorflow.org/datasets/catalog/overview</li> <li>http://deeplearning.net/datasets/</li> <li>https://deeplearning4j.org/opendata</li> <li>http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html</li> <li>https://paperswithcode.com/datasets</li> </ul> <p>Extreme classification:</p> <ul> <li>http://manikvarma.org/downloads/XC/XMLRepository.html</li> </ul> <p>MLData (down)</p> <ul> <li>http://mldata.org/</li> </ul> <p>AutoWEKA datasets:</p> <ul> <li>http://www.cs.ubc.ca/labs/beta/Projects/autoweka/datasets/</li> </ul> <p>Kaggle public datasets</p> <ul> <li>https://www.kaggle.com/datasets</li> </ul> <p>RAMP Challenge datasets</p> <ul> <li>http://www.ramp.studio/data_domains</li> </ul> <p>Wolfram data repository</p> <ul> <li>http://datarepository.wolframcloud.com/</li> </ul> <p>Data.world</p> <ul> <li>https://data.world/</li> </ul> <p>Figshare (needs digging, lots of Excel files)</p> <ul> <li>https://figshare.com/search?q=dataset&amp;quick=1</li> </ul> <p>KDNuggets list of data sets (meta-list, lots of stuff here):</p> <ul> <li>http://www.kdnuggets.com/datasets/index.html</li> </ul> <p>Benchmark Data Sets for Highly Imbalanced Binary Classification</p> <ul> <li>http://www.cs.gsu.edu/~zding/research/imbalance-data/x19data.txt</li> </ul> <p>Feature Selection Challenge Datasets</p> <ul> <li>http://www.nipsfsc.ecs.soton.ac.uk/datasets/</li> <li>http://featureselection.asu.edu/datasets.php</li> </ul> <p>BigML's list of 1000+ data sources</p> <ul> <li>http://blog.bigml.com/2013/02/28/data-data-data-thousands-of-public-data-sources/</li> </ul> <p>Massive list from Data Science Central.</p> <ul> <li>http://www.datasciencecentral.com/profiles/blogs/data-sources-for-cool-data-science-projects</li> </ul> <p>R packages (also see https://github.com/openml/openml-r/issues/185)</p> <ul> <li>http://stat.ethz.ch/R-manual/R-patched/library/datasets/html/00Index.html</li> <li>mlbench</li> <li>Stata datasets: http://www.stata-press.com/data/r13/r.html</li> </ul> <p>UTwente Activity recognition datasets:</p> <ul> <li>http://ps.ewi.utwente.nl/Datasets.php</li> </ul> <p>Vanderbilt:</p> <ul> <li>http://biostat.mc.vanderbilt.edu/wiki/Main/DataSets</li> </ul> <p>Quandl</p> <ul> <li>https://www.quandl.com</li> </ul> <p>Microarray data:</p> <ul> <li>http://genomics-pubs.princeton.edu/oncology/</li> <li>http://svitsrv25.epfl.ch/R-doc/library/multtest/html/golub.html</li> </ul> <p>Medical data:</p> <ul> <li>http://www.healthdata.gov/</li> <li>http://homepages.inf.ed.ac.uk/rbf/IAPR/researchers/PPRPAGES/pprdat.htm</li> <li>http://hcup-us.ahrq.gov/</li> <li>https://www.cms.gov/Research-Statistics-Data-and-Systems/Statistics-Trends-and-Reports/Medicare-Provider-Charge-Data/Physician-and-Other-Supplier.html</li> <li>https://nsduhweb.rti.org/respweb/homepage.cfm</li> <li>http://orwh.od.nih.gov/resources/policyreports/womenofcolor.asp</li> </ul> <p>Nature.com Scientific data repositories list</p> <ul> <li>https://www.nature.com/sdata/policies/repositories</li> </ul>"},{"location":"contributing/backend/Java-App/","title":"Evaluation Engine","text":"<p>Phasing out</p> <p>This documentation is about the older Java-based version of the OpenML evaluation engine, which will be phased out. These parts are being rewritten as a set of independent services in Python.</p> <p>When you submit datasets or experiments (runs) to OpenML, they will be processed by set of server-side processes, combined in the 'Evaluation Engine':</p> <ul> <li>It extracts the features in tabular datasets and their statistical types</li> <li>It computes a set of dataset characteristics (meta-features), e.g. the number of features and classes, that help with search and filtering, or to compute dataset similarity measures</li> <li>It evaluates experiments using a set of server-side evaluation metrics that are computed uniformly for all experiments so that they are comparable</li> <li>It creates consistent train-test splits based on task characteristics.</li> </ul> <p>The application that implements the evaluation engine was originally implemented in Java because it bulds on the Weka API. It is invoked from the OpenML API by means of a CLI interface. Typically, a call looks like this:</p> <p><code>java -jar webapplication.jar -config \"api_key=S3CR3T_AP1_K3Y\" -f evaluate_run -r 500</code></p> <p>Which in this case executes the webapplication jar, invokes the function \"evaluate run\" and gives it parameter run id 500. The config parameter can be used to set some config items, in this case the api_key is mandatory. Every OpenML user has an api_key, which can be downloaded from their OpenML profile page. The response of this function is a call to the OpenML API uploading evaluation results to the OpenML database. Note that in this case the PHP website invokes the Java webapplication, which makes a call to the PHP website again, albeit another endpoint. </p> <p>The webapplication does not have direct writing rights into the database. All communication to the database goes by means of the OpenML Connector, which communicates with the OpenML API. As a consequence, the webapplication could run on any system, i.e., there is no formal need for the webapplication to be on the same server as the website code. This is important, since this created modularity, and not all servers provide a command line interface to PHP scripts.</p> <p>Another example is the following:</p> <p><code>java -jar webapplication -config \"api_key=S3CR3T_AP1_K3Y\" -f all_wrong -r 81,161 -t 59</code></p> <p>Which takes a comma separated list of run ids (no spaces) and a task id as input and outputs the test examples on the dataset on which all algorithms used in the runs produced wrong examples (in this case, weka.BayesNet_K2 and weka.SMO, respectively). An error will be displayed if there are runs not consistent with the task id in there. </p>"},{"location":"contributing/backend/Java-App/#extending-the-java-app","title":"Extending the Java App","text":"<p>The bootstrap class of the webapplication is</p> <p><code>org.openml.webapplication.Main</code></p> <p>It automatically checks authentication settings (such as api_key) and the determines which function to invoke. </p> <p>It uses a switch-like if - else contruction to facilitate the functionalities of the various functions. Additional functions can be added to this freely. From there on, it is easy to add functionality to the webapplication. </p> <p>Parameters are handled using the Apache Commons CommandLineParser class, which makes sure that the passed parameters are available to the program. </p> <p>In order to make new functionalities available to the website, there also needs to be programmed an interface to the function, somewhere in the website. The next section details on that. </p>"},{"location":"contributing/backend/Java-App/#interfacing-from-the-openml-api","title":"Interfacing from the OpenML API","text":"<p>By design, the REST API is not allowed to communicate with the Java App. All interfaces with the Java webapplication should go through other controllers of the PHP CodeIgniter framework., for example api_splits. Currently, the website features two main API's. These are represented by a Controller. Controllers can be found in the folder openml_OS/controllers. Here we see: * api_new.php, representing the REST API * api_splits.php, representing an API interfacing to the Java webapplication. </p>"},{"location":"contributing/backend/Java-App/#helper-functions","title":"Helper functions","text":"<p>The Java code is available in the 'OpenML' repository: https://github.com/openml/OpenML/tree/master/Java</p>"},{"location":"contributing/backend/Java-App/#components","title":"Components","text":"<p>Support for tasks:  </p> <ul> <li>foldgeneration: Java code for generating cross-validation folds. Can be used from command line.</li> <li>splitgeneration: Split generator for cross validation and holdout. Unsure what's the difference with the previous?</li> <li>generate_predictions: Helper class to build prediction files based on WEKA output. Move to WEKA repository?</li> <li>evaluate_predictions: The evaluation engine computing evaluation scores based on submitted predictions</li> </ul>"},{"location":"contributing/backend/Local-Installation/","title":"Local Installation","text":"<p>Test server</p> <p>OpenML has a fully functional test server accessible at <code>test.openml.org</code> that you can use to develop against. For many cases, this is sufficient for development, and a full local installation is not required.</p> <p>Backend evolution</p> <p>OpenML has grown organically, since before the current ecosystem of python tools for platform building. We are currently rewriting the entire backend using state-of-the-art Python tools (e.g. FastAPI) so that the entire platform can be easily installed locally in one go. We plan this to be available early/mid 2025. Please get in touch  if you want to know more or want to contribute.</p>"},{"location":"contributing/backend/Local-Installation/#using-docker-compose","title":"Using Docker Compose","text":"<p>The easiest way to set up a local version of OpenML is to use Docker Compose following the instructions here (thanks to Jos van der Velde!): https://github.com/openml/services.</p> <p>If you run into problems, please post an issue in the same github repo.</p>"},{"location":"contributing/backend/Local-Installation/#installation-from-scratch","title":"Installation from scratch","text":"<p>If you want to install a local version of OpenML from scratch please follow the steps mentioned below. Note that this does not include the Kubernetes and S3 Object storage components that we use in production.</p>"},{"location":"contributing/backend/Local-Installation/#requirements","title":"Requirements","text":"<p>You'll need to have the following software running: * Apache Webserver, (with the rewrite module enabled. Is installed by default, not enabled.) * MySQL Server. * PHP 5.5 or higher (comes also with Apache) Or just a XAMP (Mac), LAMP (Linux) or WAMP (Windows) package, which conveniently contains all these applications.</p>"},{"location":"contributing/backend/Local-Installation/#databases","title":"Databases","text":"<p>Next, OpenML runs on two databases, a public database with all experiment information, and a private database, with information like user accounts etc. The latest version of both databases can be downloaded here: https://docs.openml.org/resources</p> <p>Obviously, the private database does not include any actual user account info.</p>"},{"location":"contributing/backend/Local-Installation/#backend","title":"Backend","text":"<p>The source code is available in the 'OpenML' repository: https://github.com/openml/OpenML</p> <p>OpenML is written in PHP, and can be 'installed' by copying all files in the 'www' or 'public_html' directory of Apache.</p> <p>After that, you need to provide your local paths and database accounts and passwords using the config file in: 'APACHE_WWW_DIR'/openml_OS/config/BASE_CONFIG.php.</p> <p>If everything is configured correctly, OpenML should now be running.</p>"},{"location":"contributing/backend/Local-Installation/#search-indices","title":"Search Indices","text":"<p>If you want to run your own (separate) OpenML instance, and store your own data, you'll also want to build your own search indices to show all data on the website. The OpenML website is based on the ElasticSearch stack. To install it, follow the instructions here: http://knowm.org/how-to-set-up-the-elk-stack-elasticsearch-logstash-and-kibana/</p>"},{"location":"contributing/backend/Local-Installation/#initialization","title":"Initialization","text":"<p>This script wipes all OpenML server data and rebuilds the database and search index. Replace 'openmldir' with the directory where you want OpenML to store files.</p> <pre><code># delete data from server\nsudo rm -rf /openmldir/*\nmkdir /openmldir/log\n\n# delete database\nmysqladmin -u \"root\" -p\"yourpassword\" DROP openml_expdb\nmysql -h localhost -u root -p\"yourpassword\" -e \"TRUNCATE openml.file;\"\n\n# reset ES search index\necho \"Deleting and recreating the ES index: \"\ncurl -XDELETE http://localhost:9200/openml\ncurl -XPUT 'localhost:9200/openml?pretty' -H 'Content-Type: application/json' -d'\n{\n    \"settings\" : {\n        \"index\" : {\n            \"number_of_shards\" : 3,\n            \"number_of_replicas\" : 2\n        }\n    }\n}\n'\n\n# go to directory with the website source code\ncd /var/www/openml.org/public_html/\n\n# reinitiate the database\nmysql -u root -p\"yourpassword!\" &lt; downloads/openml_expdb.sql\n\n# fill important columns\nsudo php index.php cron install_database\n\n# rebuild search index\nsudo php index.php cron initialize_es_indices\nsudo php index.php cron build_es_indices\n\nsudo chown apache:apache /openmldir/log\nsudo chown apache:apache /openmldir/log/*\n</code></pre>"},{"location":"contributing/clients/Client-API-Standards/","title":"Client development","text":""},{"location":"contributing/clients/Client-API-Standards/#building-clients","title":"Building clients","text":"<p>You can access OpenML datasets, pipelines, benchmarks, and much more, through a range of client APIs. Well-developed clients exist in Python, R, Java, and several other languages. Please see their documentation (in the other tabs) for more guidance of how to contribute to them.</p> <p>If you want to develop your own client (e.g. for a new language), please check out the following resources:  </p> <ul> <li>REST API: all endpoints to GET, POST, or DELETE resources</li> <li>Metadata Standard: how we describe datasets and all other OpenML resources</li> <li>Minimal standards (below) for uniform client configuration and caching mechanisms, to make the client behavior more uniform across languages.</li> </ul> <p>Integrating tools</p> <p>If you want to integrate OpenML into machine learning and data science tools, it's often easier to build on one of the existing clients,  which often can be used as is or extended. For instance, see how to extend the Python API to integrate OpenML into Python tools. </p>"},{"location":"contributing/clients/Client-API-Standards/#minimal-standards","title":"Minimal standards","text":""},{"location":"contributing/clients/Client-API-Standards/#configuration-file","title":"Configuration file","text":"<p>The configuration file resides in a directory <code>.openml</code> in the home directory of the user and is called config. It consists of <code>key = value</code> pairs which are seperated by newlines. The following keys are defined:</p> <ul> <li>apikey:<ul> <li>required to access the server</li> </ul> </li> <li>server:<ul> <li>default: <code>http://www.openml.org</code></li> </ul> </li> <li>verbosity:<ul> <li>0: normal output</li> <li>1: info output</li> <li>2: debug output</li> </ul> </li> <li>cachedir:<ul> <li>if not given, will default to <code>file.path(tempdir(), \"cache\")</code>.</li> </ul> </li> </ul>"},{"location":"contributing/clients/Client-API-Standards/#caching","title":"Caching","text":""},{"location":"contributing/clients/Client-API-Standards/#cache-invalidation","title":"Cache invalidation","text":"<p>All parts of the entities which affect experiments are immutable. The entities dataset and task have a flag <code>status</code> which tells the user whether they can be used safely.</p>"},{"location":"contributing/clients/Client-API-Standards/#file-structure","title":"File structure","text":"<p>Caching should be implemented for</p> <ul> <li>datasets</li> <li>tasks</li> <li>splits</li> <li>predictions</li> </ul> <p>and further entities might follow in the future. The cache directory <code>$cache</code> should be specified by the user when invoking the API. The structure in the cache directory should be as following:</p> <ul> <li>One directory for the following entities:<ul> <li><code>$cache/datasets</code></li> <li><code>$cache/tasks</code></li> <li><code>$cache/runs</code></li> </ul> </li> <li>For every dataset there is an extra directory for which the name is the dataset ID, e.g. <code>$cache/datasets/2</code> for the dataset with OpenML ID 2.<ul> <li>The dataset should be called <code>dataset.pq</code> or <code>dataset.arff</code></li> <li>Every other file should be named by the API call which was used to obtain it. The XML returned by invoking <code>openml.data.qualities</code> should therefore be called qualities.xml.</li> </ul> </li> <li>For every task there is an extra directory for which the name is the task ID, e.g. <code>$cache/tasks/1</code><ul> <li>The task file should be called <code>task.xml</code>.</li> <li>The splits accompanying a task are stored in a file <code>datasplits.arff</code>.</li> </ul> </li> <li>For every run there is an extra directory for which the name is the run ID, e.g. <code>$cache/run/1</code><ul> <li>The predictions should be called <code>predictions.arff</code>.</li> </ul> </li> </ul>"},{"location":"contributing/clients/Rest/","title":"REST API","text":"<p>OpenML offers a RESTful Web API, with predictive URLs, for uploading and downloading machine learning resources. Try the REST API Documentation to see examples of all calls, and test them right in your browser.</p>"},{"location":"contributing/clients/Rest/#getting-started","title":"Getting started","text":"<ul> <li>REST services can be called using simple HTTP GET or POST actions.</li> <li>The REST Endpoint URL is <code>https://www.openml.org/api/v1/</code></li> <li>The default endpoint returns data in XML. If you prefer JSON, use the endpoint <code>https://www.openml.org/api/v1/json/</code>. Note that, to upload content, you still need to use XML (at least for now).</li> </ul>"},{"location":"contributing/clients/Rest/#testing","title":"Testing","text":"<p>For continuous integration and testing purposes, we have a test server offering the same API, but which does not affect the production server.</p> <ul> <li>The test server REST Endpoint URL is <code>https://test.openml.org/api/v1/</code></li> </ul>"},{"location":"contributing/clients/Rest/#error-messages","title":"Error messages","text":"<p>Error messages will look like this:</p> <pre><code>&lt;oml:error xmlns:oml=\"http://openml.org/error\"&gt;\n&lt;oml:code&gt;100&lt;/oml:code&gt;\n&lt;oml:message&gt;Please invoke legal function&lt;/oml:message&gt;\n&lt;oml:additional_information&gt;Additional information, not always available.&lt;/oml:additional_information&gt;\n&lt;/oml:error&gt;\n</code></pre> <p>All error messages are listed in the API documentation. E.g. try to get a non-existing dataset:</p> <ul> <li>in XML: https://www.openml.org/api_new/v1/data/99999</li> <li>in JSON: https://www.openml.org/api_new/v1/json/data/99999</li> </ul>"},{"location":"contributing/clients/Rest/#examples","title":"Examples","text":"<p>You need to be logged in for these examples to work.</p>"},{"location":"contributing/clients/Rest/#download-a-dataset","title":"Download a dataset","text":"<ul> <li>User asks for a dataset using the /data/{id} service. The <code>dataset id</code> is typically part of a task, or can be found on OpenML.org.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting the dataset. This can be OpenML, but also any other data repository. Try it now</li> </ul>"},{"location":"contributing/clients/Rest/#download-a-flow","title":"Download a flow","text":"<ul> <li>User asks for a flow using the /flow/{id} service and a <code>flow id</code>. The <code>flow id</code> can be found on OpenML.org.</li> <li>OpenML returns a description of the flow as an XML file (or JSON). Try it now</li> <li>The flow description contains the URL where the flow can be downloaded (e.g. GitHub), either as source, binary or both, as well as additional information on history, dependencies and licence. The user calls the right URL to download it.</li> <li>The flow is returned by the server hosting it. This can be OpenML, but also any other code repository. Try it now</li> </ul>"},{"location":"contributing/clients/Rest/#download-a-task","title":"Download a task","text":"<ul> <li>User asks for a task using the /task/{id} service and a <code>task id</code>. The <code>task id</code> is typically returned when searching for tasks.</li> <li>OpenML returns a description of the task as an XML file (or JSON). Try it now</li> <li>The task description contains the <code>dataset id</code>(s) of the datasets involved in this task. The user asks for the dataset using the /data/{id} service and the <code>dataset id</code>.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting it. This can be OpenML, but also any other data repository. Try it now</li> <li>The task description may also contain links to other resources, such as the train-test splits to be used in cross-validation. The user calls that URL to download the train-test splits.</li> <li>The train-test splits are returned by OpenML. Try it now</li> </ul>"},{"location":"contributing/clients/metadata_definition/","title":"Metadata definition","text":"<p>OpenML is at its core a meta-database, from which datasets, pipelines (flows), experiments (runs) and other entities can be downloaded and uploaded, all described using a clearly defined meta-data standard. In this document, we describe the standard how to upload entities to OpenML and what the resulting database state will be.</p> <p> Croissant</p> <p>OpenML has partnered with MLCommons, Google, Kaggle, HuggingFace, and a consortium of other partners to define a new metadata standard for machine learning datasets:  Croissant! You can already download all OpenML datasets in the Croissant format, and we're working further supporting and extending Croissant.</p> <p>Below is the OpenML metadata standard for version 1 of the API.</p>"},{"location":"contributing/clients/metadata_definition/#data","title":"Data","text":"<p>Data is uploaded through the function post data. The following files are needed:</p> <ul> <li><code>description</code>: An XML adhiring to the XSD schema.</li> <li><code>dataset</code>: An ARFF file containing the data (optional, if not set, there should be an URL in the description, pointing to this file).   Uploading any other files will result in an error.</li> </ul>"},{"location":"contributing/clients/metadata_definition/#tasks","title":"Tasks","text":"<p>Tasks are uploaded through the function post task. The following files are needed:</p> <ul> <li><code>description</code>: An XML adhering to the XSD schema.   Uploading any other files will result in an error.</li> </ul> <p>The task file should contain several input fields. These are a name and value combination of fields that are marked to be relevant by the task type definition. There are several task type definitions, e.g.:</p> <ul> <li>Supervised Classification</li> <li>Supervised Regression</li> <li>Learning Curve</li> <li>Data Stream Classification</li> </ul> <p>Note that the task types themselves are flexible content (ideally users can contribute task types) and therefore the documents are not part of the OpenML definition. The task types define which input fields should be set, when creating a task.</p> <p>Duplicate tasks (i.e., same value for <code>task_type_id</code> and all <code>input</code> fields equal) will be rejected.</p> <p>When creating a task, the API checks for all of the input fields whether the input is legitimate. (Todo: describe the checks and what they depend on).</p>"},{"location":"contributing/clients/metadata_definition/#flow","title":"Flow","text":"<p>Flows are uploaded through the function post flow. The following file is needed:</p> <ul> <li><code>description</code>: An XML adhering to the XSD schema.   Uploading any other files will result in an error.</li> </ul> <p>Duplicate flows (i.e., same values for <code>name</code> and <code>external_version</code>) will be rejected.</p>"},{"location":"contributing/clients/metadata_definition/#runs","title":"Runs","text":"<p>Runs are uploaded through the function post run. The following files are needed:</p> <ul> <li><code>description</code>: An XML adhering to the XSD schema.</li> <li><code>predictions</code>: An ARFF file containing the predictions (optional, depending on the task).</li> <li><code>trace</code>: An ARFF file containing the run trace (optional, depending on the flow).   Uploading any other files will result in an error.</li> </ul>"},{"location":"contributing/clients/metadata_definition/#predictions","title":"Predictions","text":"<p>The contents of the prediction file depends on the task type.</p>"},{"location":"contributing/clients/metadata_definition/#task-type-supervised-classification","title":"Task type: Supervised classification","text":"<p>Example predictions file</p> <ul> <li>repeat NUMERIC</li> <li>fold NUMERIC</li> <li>row_id NUMERIC</li> <li>confidence.{$classname}: optional. various columns, describing the confidence per class. The values of these columns should add to 1 (precision 1e-6).</li> <li>(proposal) decision_function.{$classname}: optional. various columns, describing decision function per class.</li> <li>prediction {$classname}   Runs that have a different set of columns will be rejected.</li> </ul>"},{"location":"contributing/clients/metadata_definition/#trace","title":"Trace","text":"<p>Example trace file</p> <ul> <li>repeat: cross-validation repeat</li> <li>fold: cross-validation fold</li> <li>iteration: the index order within this repeat/fold combination</li> <li>evaluation (float): the evaluation score that was attached based on the validation set</li> <li>selected {True, False}: Whether in this repeat/run combination this was the selected hyperparameter configuration (exactly one should be tagged with True)</li> <li>Per optimized parameter a column that has the name of the parameter and the prefix \"parameter_\"</li> <li>setup_string: Due to legacy reasons accepted, but will be ignored by the default evaluation engine</li> </ul> <p>Traces that have a different set of columns will be rejected.</p>"},{"location":"contributing/website/Dash/","title":"Dash visualization","text":"<p>Dash is a python framework which is suitable for building data visualization dashboards using pure python. Dash is written on top of plotly, react and flask and the graphs are defined using plotly python. The dash application is composed of two major parts :</p> <ul> <li><code>Layout</code> - Describes how the dashboard looks like</li> <li><code>Callbacks</code> - Used to update graphs, tables in the layout and makes the dashboard interactive.</li> </ul>"},{"location":"contributing/website/Dash/#files","title":"Files","text":"<p>The dash application is organized as follows:</p> <ul> <li> <p><code>dashapp.py</code></p> </li> <li> <p>Creates the dash application</p> </li> <li>The dash app is embedded in the flask app passed to <code>create_dash_app</code> function</li> <li> <p>This file need not be modified to create a new plot</p> </li> <li> <p><code>layouts.py</code></p> </li> <li> <p>contains the layout for all the pages</p> </li> <li><code>get_layout_from_data</code>- returns layout of data visualization</li> <li><code>get_layout_from_task</code>- returns layout of taskvisualization</li> <li><code>get_layout_from_flow</code>- returns layout of flow visualization</li> <li><code>get_layout_from_run</code> - returns layout of run visualization</li> <li> <p>This file needs to be modified to add a new plot (data, task, flow, run)</p> </li> <li> <p><code>callbacks.py</code></p> </li> <li>Registers all the callbacks for the dash application</li> <li>This file needs to be modified to add a new plot, especially if the plot needs to be interactive</li> </ul>"},{"location":"contributing/website/Dash/#how-the-dashboard-works","title":"How the dashboard works","text":"<p>In this dash application, we need to create the layout of the page dynamically based on the entered URL. For example, [http://127.0.0.1:5000/dashboard/data/5] needs to return the layout for dataset id #5 whereas [http://127.0.0.1:5000/dashboard/run/5] needs to return the layout for run id #5.</p> <p>Hence , the dash app is initially created with a dummy <code>app.layout</code> by dashapp.py and the callbacks are registered for the app using <code>register_callbacks</code> function.</p> <ul> <li> <p>render_layout is the callback which dynamically renders layout. Once the dash app is running, the first callback which is fired is <code>render_layout.</code>   This is the main callback invoked when a URL with a data , task, run or flow ID is entered.   Based on the information in the URL, this method returns the layout.</p> </li> <li> <p>Based on the URL, get_layout_from_data, get_layout_from_task, get_layout_from_flow, get_layout_from_run are called.   These functions define the layout of the page - tables, html Divs, tabs, graphs etc.</p> </li> <li> <p>The callbacks corresponding to each component in the layout are invoked to update the components dynamically and   make the graphs interactive. For example, update_scatter_plot in <code>data_callbacks.py</code> updates the scatter plot   component in the data visualization dashboard.</p> </li> </ul>"},{"location":"contributing/website/Flask/","title":"Flask backend","text":"<p>We use Flask as our web framework. It handles user authentication, dataset upload, task creation, and other aspects that require server-side interaction. It is designed to be independent from the OpenML API. This means that you can use it to create your own personal frontend for OpenML, using the main OpenML server to provide the data. Of course, you can also link it to your own local OpenML setup.</p>"},{"location":"contributing/website/Flask/#design","title":"Design","text":"<p>Out flask app follows Application factories design pattern. A new app instance can be created by: <pre><code>    from autoapp import create_app\n    app = create_app(config_object)\n</code></pre></p> <p>The backend is designed in a modular fashion with flask Blueprints. Currently, the flask app consists of two blueprints public and user:</p> <ul><li>Public blueprint: contains routes that do not require user authentication or authorization. like signup and forgot password.</li> <li>User blueprint: Contains routes which require user authentication like login, changes in profile and fetching API key.</li></ul> <p>New blueprints can be registered in `server/app.py` with register_blueprints function:</p> <pre><code>    def register_blueprints(app):\n        app.register_blueprint(new_blueprint)\n</code></pre>"},{"location":"contributing/website/Flask/#database-setup","title":"Database setup","text":"<p>If you want o setup a local user database similar to OpenML then follow these steps:</p> <ol> <li>Install MySQL</li> <li>Create a new database 'openml'</li> <li>Set current database to 'openml' via use method</li> <li>Download users.sql file from openml.org github repo and add it in the openml db via \"mysql -u root -p openml &lt; users.sql\"</li> <li>Edit the database path in `server/extensions.py` and `server/config.py`</li> </ol> <p>Note: Remember to add passwords and socket extension address(if any) in both in <code>server/extensions.py</code> and <code>server/config.py</code> </p>"},{"location":"contributing/website/Flask/#security","title":"Security","text":"<p>Flask backend uses JSON web tokens for all the user handling tasks. Flask JWT extended library is used to bind JWT with the flask app. Current Mechanism is :</p> <ol> <li> User logs in.</li> <li> JWT token is assigned to user and sent with every request to frontend.</li> <li> All the user information can only be accessed with a JWT token like edit profile and API-key.</li> <li> The JWT token is stored in local memory of the browser.</li> <li> The token get expired after 2 hours or get blacklisted after logout.</li> </ol> <p>JWT is registered as an extension in `server/extensions.py`. All the user password hash are saved in Argon2 format with the new backend.</p>"},{"location":"contributing/website/Flask/#registering-extensions","title":"Registering Extensions","text":"<p>To register a new extension to flask backend extension has to be added in <code>server/extensions.py</code> and initialized in server/app.py. Current extensions are : flask_argon2, flask_bcrypt, flask_jwt_extended and flask_sqlalchemy.</p>"},{"location":"contributing/website/Flask/#configuring-app","title":"Configuring App","text":"<p>Configuration variables like secret keys, Database URI and extension configurations are specified in  <code>server/config.py</code> with Config object, which is supplied to the flask app during initialization.</p>"},{"location":"contributing/website/Flask/#creating-a-new-route","title":"Creating a new route","text":"<p>To create a new route in backend you can add the route in <code>server/public/views.py</code> or <code>server/user/views.py</code> (if it requires user authorisation or JWT usage in any way).  </p>"},{"location":"contributing/website/Flask/#bindings-to-openml-server","title":"Bindings to OpenML server","text":"<p>You can specify which OpenML server to connect to. This is stored in the <code>.env</code> file in the main directory. It is set to the main OpenML server by default:</p> <pre><code>    ELASTICSEARCH_SERVER=https://www.openml.org/es\n    OPENML_SERVER=https://www.openml.org\n</code></pre> <p>The ElasticSearch server is used to download information about datasets, tasks, flows and runs, as well as to power the frontend search. The OpenML server is used for uploading datasets, tasks, and anything else that requires calls to the OpenML API.</p>"},{"location":"contributing/website/Flask/#bindings-to-frontend","title":"Bindings to frontend","text":"<p>The frontend is generated by React. See below for more information. The React app is loaded as a static website. This is done in Flask setup in file <code>server.py</code>.</p> <pre><code>    app = Flask(__name__, static_url_path='', static_folder='src/client/app/build')\n</code></pre> <p>It will find the React app there and load it.</p>"},{"location":"contributing/website/Flask/#email-server","title":"Email Server","text":"<p>OpenML uses its own mail server, You can use basically any mail server compatible with python SMTP library. Our suggestion is to use mailtrap.io for local testing. You can configure email server configurations in .env file. Currently we only use emails for confirmation email and forgotten password emails.</p>"},{"location":"contributing/website/React/","title":"React App","text":""},{"location":"contributing/website/React/#app-structure","title":"App structure","text":"<p>The structure of the source code looks as follows</p> <pre><code>App.js\nindex.js\ncomponents\n|-- Sidebar.js\n|-- Header.js\n|-- ...\nlayouts\n|-- Clear.js\n|-- Main.js\npages\n|-- auth\n|-- cover\n|-- docs\n|-- search\nroutes\n|-- index.js\n|-- Routes.js\nthemes\n</code></pre> <p>The website is designed as a single-page application. The top level files bootstrap the app. <code>index.js</code> simply renders the top component, and <code>App.js</code> adds the relevant subcomponents based on the current theme and state.</p> <p><code>Routes.js</code> links components to the possible routes (based on the URL). The list of possible routes is defined in <code>routes/index.js</code>.</p> <p><code>pages</code> contain the various pages of the website. It has subdirectories for:</p> <ul> <li><code>auth</code>: All pages that require authorization (login). These routes are protected.</li> <li><code>cover</code>: The front page of the website</li> <li><code>docs</code>: All normal information pages (e.g. 'About', 'API',...)</li> <li><code>search</code>: All pages related to searching for datasets, tasks, flows, runs, etc.</li> </ul> <p><code>layout</code> contains the possible layouts, <code>Main</code> or <code>Clear</code> (see below). You define the layout of a page by adding its route to either <code>mainRoutes</code> or <code>clearRoutes</code> in <code>routes/index.js</code>. The default is the <code>Main</code> layout.</p> <p><code>themes</code> contains the overall theme styling for the entire website. Currently, there is a dark and a light theme. They can be set using <code>setTheme</code> in the MainContext, see <code>App.js</code>.</p>"},{"location":"contributing/website/React/#component-structure","title":"Component structure","text":"<p>The component structure is shown above, for the <code>Main</code> layout. The <code>App</code> component also holds the state of the website using React's native Context API (see below). Next to the header and sidebar, the main component of the website (in yellow) shows the contents of the current <code>page</code>. In this image, this is the search page, which has several subcomponents as explained below.</p>"},{"location":"contributing/website/React/#search-page","title":"Search page","text":"<p>The search page is structured as follows:</p> <ul> <li> <p><code>SearchPanel</code>: the main search panel. Also contains callbacks for sorting and filtering, and lists what can be filtered or sorted on.</p> </li> <li> <p><code>FilterBar</code>: The top bar with the search statistics and functionality to add filters and sort results</p> </li> <li> <p><code>SearchResultsPanel</code>: The list of search results on the left. It shows a list of <code>Card</code> elements which are uniformly styled but their contents may vary. Depending on the selected type of result (selected in the left navigation bar) it is instantiated with different properties. E.g. a <code>DataListPanel</code> is a simple wrapper around <code>SearchResultsPanel</code> which defines the dataset-specific statistics to be shown in the cards.</p> <ul> <li>Search tabs: The tabs that allow you to choose between different aspects of the results (Statistics, Overview (Dash)) or the different views on the selected dataset, task, etc. (Details, Analysis (Dash),...)</li> <li><code>ItemDetail</code>: When a search result is selected, this will show the details of the selection, e.g. the dataset details. Depending on the passed <code>type</code> prop, it will render the <code>Dataset</code>, <code>Task</code>, ... component.</li> </ul> </li> </ul> <p>The <code>api.js</code> file contains the <code>search</code> function, which translates a search query, filters, and other constraints into an ElasticSearch query and returns the results.</p>"},{"location":"contributing/website/React/#style-guide","title":"Style guide","text":"<p>To keep a consistent style and minimize dependencies and complexity, we build on Material UI components and FontAwesome icons. Theming is defined in <code>themes/index.js</code> and loaded in as a context (<code>ThemeContext</code>) in <code>App.js</code>. More specific styling is always defined through styled components in the corresponding pages.</p>"},{"location":"contributing/website/React/#layouts","title":"Layouts","text":"<p>There are two top level layouts: <code>Main</code> loads the main layout with a <code>Sidebar</code>, <code>Header</code>, and a certain page with all the contents. The <code>Clear.js</code> layout has no headers or sidebars, but has a colored gradient background. It is used mainly for user login and registration or other quick forms.</p> <p>The layout of the page content should use the Material UI grid layout. This makes sure it will adapt to different device screen sizes. Test using your browsers development tools whether the layout adapts correctly to different screens, including recent smartphones.</p>"},{"location":"contributing/website/React/#styled-components","title":"Styled components","text":"<p>Any custom styling (beyond the Material UI default styling) is defined in styled components which are defined within the file for each page. Keep this as minimal as possible. Check if you can import styled components already defined for other pages, avoid duplication.</p> <p>Styled div's are defined as follows:</p> <pre><code>const OpenMLTitle = styled.div`\n  color: white;\n  font-size: 3em;\n`;\n</code></pre> <p>Material UI components can be styled the same way:</p> <pre><code>const WhiteButton = styled(Button)`\n  display: inline-block;\n  color: #fff;\n`;\n</code></pre>"},{"location":"contributing/website/React/#color-palette","title":"Color palette","text":"<p>We follow the general Material UI color palette with shade 400, except when that doesn't give sufficient contrast. The main colors used (e.g. for the icons in the sidebar are: 'green[400]', 'yellow[700]', 'blue[800]', 'red[400]', 'purple[400]', 'orange[400]', 'grey[400]'. Backgrounds are generally kept white (or dark grey for the dark theme). The global context (see below) has a <code>getColor</code> function to get the colors of the search types, e.g. <code>context.getColor(\"run\")</code> returns <code>red[400]</code>.</p>"},{"location":"contributing/website/React/#handling-state","title":"Handling state","text":"<p>There are different levels of state management:</p> <ul> <li>Global state is handled via React's native Context API (we don't use Redux). Contexts are defined in the component tree where needed (usually higher up) by a context provider component, and is accessed lower in the component tree by a context consumer. For instance, see the <code>ThemeContext.Provider</code> in <code>App.js</code> and the <code>ThemeContext.Consumer</code> in <code>Sidebar.js</code>. There is a <code>MainContext</code> which contains global state values such as the logged in user details, and the current state of the search.</li> <li>Lower level components can pass state to their child components via props.</li> <li>Local state changes should, when possible, be defined by React Hooks.</li> </ul> <p>Note that changing the global state will re-render the entire website. Hence, do this only when necessary.</p>"},{"location":"contributing/website/React/#state-and-search","title":"State and search","text":"<p>Most global state variables have to do with search. The search pages typically work by changing the <code>query</code> and <code>filters</code> variables (see <code>App.js</code>). There is a <code>setSearch</code> function in the main context that can be called to change the search parameters. It checks whether the query has changed and whether updating the global state and re-rendering the website is necessary.</p>"},{"location":"contributing/website/React/#lifecycle-methods","title":"Lifecycle Methods","text":"<p>These are the React lifecycle methods and how we use them. When a component mounts, methods 1,2,4,7 will be called. When it updates, methods 2-6 will be called.</p> <ol> <li>constructor(): Set the initial state of the components</li> <li>getDerivedStateFromProps(props, state): Static method, only for changing the local state based on props. It returns the new state.</li> <li>shouldComponentUpdate(nextProps, nextState): Decides whether a state change requires a re-rendering or not. Used to optimize performance.</li> <li>render(): Returns the JSX to be rendered. It should NOT change the state.</li> <li>getSnapshotBeforeUpdate(prevProps,prevState): Used to save 'old' DOM information right before an update. Returns a 'snapshot'.</li> <li>componentDidUpdate(prevProps,prevState,snapshot): For async requests or other operations right after component update.</li> <li>componentDidMount(): For async requests (e.g. API calls) right after the component mounted.</li> <li>componentWillUnMount(): Cleanup before the component is destroyed.</li> <li>componentDidCatch(error,info): For updating the state after an error is thrown.</li> </ol>"},{"location":"contributing/website/React/#forms-and-events","title":"Forms and Events","text":"<p>React wraps native browser events into synthetic events to handle interactions in a cross-browser compatible way. After being wrapped, they are sent to all event handlers, usually defined as callbacks. Note: for performance reasons, synthetic events are pooled and reused, so their properties are nullified after being consumed. If you want to use them asynchronously, you need to call <code>event.persist()</code>.</p> <p>HTML forms are different than other DOM elements because they keep their own state in plain HTML. To make sure that we can control the state we need to set the input field's <code>value</code> to a component state value.</p> <p>Here's an example of using an input field to change the title displayed in the component.</p> <pre><code>const titles: {mainTitle: 'OpenML'};\n\nclass App extends Component {\n  this.state = {titles};\n\n  // Receive synthetic event\n  onTitleChange = (event) =&gt; {\n    this.setState({titles.mainTitle : event.target.value});\n  }\n\n  render(){\n    return (\n      &lt;div classname=\"App\"&gt;\n        &lt;h1&gt;{this.state.titles.mainTitle}&lt;/h1&gt;\n        &lt;form&gt;\n          &lt;input type=\"text\"\n          value={this.state.titles.mainTitle} // control state\n          onChange={this.onTitleChange} // event handler callback\n          /&gt;\n        &lt;/form&gt;\n      &lt;/div&gt;\n    );\n  }\n}\n</code></pre>"},{"location":"contributing/website/Website/","title":"Getting started","text":""},{"location":"contributing/website/Website/#installation","title":"Installation","text":"<p>The OpenML website runs on Flask, React, and Dash. You need to install these first.</p> <ul> <li> <p>Download or clone the source code for the OpenML website from GitHub. Then, go into that folder (it should have the <code>requirements.txt</code> and <code>package.json</code> files). <pre><code>git clone https://github.com/openml/openml.org.git\ncd openml.org\n</code></pre></p> </li> <li> <p>Install Flask, Dash, and dependencies using PIP <pre><code>pip install -r requirements.txt\n</code></pre></p> </li> <li> <p>Install React and dependencies using NPM (8 or higher) <pre><code>cd server/src/client/app/\nnpm install\n</code></pre></p> </li> </ul>"},{"location":"contributing/website/Website/#building-and-running","title":"Building and running","text":"<p>Go back to the home directory. Build a production version of the website with:</p> <pre><code>npm run build --prefix server/src/client/app/\n</code></pre> <p>Start the server by running:</p> <pre><code>flask run\n</code></pre> <p>You should now see the app running in your browser at <code>localhost:5000</code></p> <p>Note: If you run the app using HTTPS, add the SSL context or use 'adhoc' to use on-the-fly certificates or you can specify your own certificates.</p> <pre><code>flask run --cert='adhoc'\n</code></pre> <p>As flask server is not suitable for production we recommend you to use some other server if you want to deploy your openml installation in production. We currently use gunicorn for production server. You can install the gunicorn server and run it: <pre><code>gunicorn --certfile cert.pem --keyfile key.pem -b localhost:5000 autoapp:app\n</code></pre></p>"},{"location":"contributing/website/Website/#development","title":"Development","text":"<p>To start the React frontend in developer mode, go to <code>server/src/client/app</code> and run:</p> <pre><code>npm run start\n</code></pre> <p>The app should automatically open at <code>localhost:3000</code> and any changes made to the code will automatically reload the website (hot loading).</p> <p>For the new Next.js frontend, install and run like this: <pre><code>cd app\nnpm install\nnpm run dev\n</code></pre></p>"},{"location":"contributing/website/Website/#structure","title":"Structure","text":"<p>The website is built on the following components:  </p> <ul> <li>A Flask backend. Written in Python, the backend takes care of all communication with the OpenML server. It builds on top of the OpenML Python API. It also takes care of user authentication and keeps the search engine (ElasticSearch) up to date with the latest information from the server. Files are located in the <code>server</code> folder.</li> <li>A React frontend. Written in JavaScript, this takes care of rendering the website. It pulls in information from the search engine, and shows plots rendered by Dash. It also contains forms (e.g. for logging in or uploading new datasets), which will be sent off to the backend for processing. Files are located in <code>server/src/client/app</code>.</li> <li>Dash dashboards. Written in Python, Dash is used for writing interactive plots. It pulls in data from the Python API, and renders the plots as React components. Files are located in <code>server/src/dashboard</code>.</li> </ul>"},{"location":"ecosystem/","title":"Ecosystem","text":"<p>OpenML has a rich ecosystem of tools and projects that seamlessly integrate OpenML in various ways. </p> <p>Add your library</p> <p>Did you use OpenML in your work and want to share it with the community? We would love to have you! Simply create a pull request with the necessary information (click the  icon) and we will add it to this page.</p> <p>Integrate OpenML in your libraries</p> <p>If you want to integrate OpenML into machine learning and data science tools, it's easiest to build on one of the existing clients,  which often can be used as is or extended. For instance, see how to extend the Python API to integrate OpenML into Python tools. </p> automlbenchmark  402 stars <p>OpenML AutoML Benchmarking Framework</p> openml-python  280 stars <p>OpenML's Python API for a World of Data and More \ud83d\udcab</p> openml-r  95 stars <p>R package to interface with OpenML</p> openml-rust  11 stars <p>A rust interface to http://openml.org/</p> OpenML.jl  10 stars <p>Partial implementation of the OpenML API for Julia</p> continual-automl  5 stars <p>Adaptations of AutoML libraries H2O, Autosklearn and GAMA for stream learning</p> openml-pytorch  5 stars <p>Pytorch extension for openml-python</p> openml-dotnet  5 stars <p>.NET API</p> openml-rapidminer  4 stars <p>RapidMiner plugin</p> openml-tensorflow  2 stars <p>Tensorflow extension for openml-python</p> OpenmlCortana  0 stars <p>Openml Cortana connector</p> openml-keras  0 stars <p>Keras extension for openml-python</p> openml-croissant  0 stars <p>Converting dataset metadata from OpenML to Croissant format</p> openml-azure  0 stars <p>Tools for interfacing with Azure</p> openml-onnx  0 stars <p>onnx extension for openml</p> openml-mxnet  0 stars <p>MXNet extension for openml</p> flow-visualization  0 stars <p>Tool to convert openml flows to ONNX and visualize them via Netron</p>"},{"location":"ecosystem/Java/","title":"Java","text":"<p>The Java API allows you connect to OpenML from Java applications.</p>"},{"location":"ecosystem/Java/#java-docs","title":"Java Docs","text":"<p>Read the full Java Docs.</p>"},{"location":"ecosystem/Java/#download","title":"Download","text":"<p>Stable releases of the Java API are available from Maven Central Or, you can check out the developer version from GitHub</p> <p>Include the jar file in your projects as usual, or install via Maven.</p>"},{"location":"ecosystem/Java/#quick-start","title":"Quick Start","text":"<ul> <li>Create an <code>OpenmlConnector</code> instance with your authentication details. This will create a client with all OpenML functionalities. <p>OpenmlConnector client = new OpenmlConnector(\"api_key\")</p> </li> </ul> <p>All functions are described in the Java Docs.</p>"},{"location":"ecosystem/Java/#downloading","title":"Downloading","text":"<p>To download data, flows, tasks, runs, etc. you need the unique id of that resource. The id is shown on each item's webpage and in the corresponding url. For instance, let's download Data set 1. The following returns a DataSetDescription object that contains all information about that data set.</p> <pre><code>DataSetDescription data = client.dataGet(1);\n</code></pre> <p>You can also search for the items you need online, and click the icon to get all id's that match a search.</p>"},{"location":"ecosystem/Java/#uploading","title":"Uploading","text":"<p>To upload data, flows, runs, etc. you need to provide a description of the object. We provide wrapper classes to provide this information, e.g. <code>DataSetDescription</code>, as well as to capture the server response, e.g. <code>UploadDataSet</code>, which always includes the generated id for reference:</p> <pre><code>DataSetDescription description = new DataSetDescription( \"iris\", \"The famous iris dataset\", \"arff\", \"class\");\nUploadDataSet result = client.dataUpload( description, datasetFile );\nint data_id = result.getId();\n</code></pre> <p>More details are given in the corresponding functions below. Also see the Java Docs for all possible inputs and return values.</p>"},{"location":"ecosystem/Java/#data-download","title":"Data download","text":""},{"location":"ecosystem/Java/#datagetint-data_id","title":"<code>dataGet(int data_id)</code>","text":"<p>Retrieves the description of a specified data set.</p> <pre><code>DataSetDescription data = client.dataGet(1);\nString name = data.getName();\nString version = data.getVersion();\nString description = data.getDescription();\nString url = data.getUrl();\n</code></pre>"},{"location":"ecosystem/Java/#datafeaturesint-data_id","title":"<code>dataFeatures(int data_id)</code>","text":"<p>Retrieves the description of the features of a specified data set.</p> <pre><code>DataFeature reponse = client.dataFeatures(1);\nDataFeature.Feature[] features = reponse.getFeatures();\nString name = features[0].getName();\nString type = features[0].getDataType();\nboolean isTarget = features[0].getIs_target();\n</code></pre>"},{"location":"ecosystem/Java/#dataqualityint-data_id","title":"<code>dataQuality(int data_id)</code>","text":"<p>Retrieves the description of the qualities (meta-features) of a specified data set.</p> <pre><code>    DataQuality response = client.dataQuality(1);\n    DataQuality.Quality[] qualities = reponse.getQualities();\n    String name = qualities[0].getName();\n    String value = qualities[0].getValue();\n</code></pre>"},{"location":"ecosystem/Java/#dataqualityint-data_id-int-start-int-end-int-interval_size","title":"<code>dataQuality(int data_id, int start, int end, int interval_size)</code>","text":"<p>For data streams. Retrieves the description of the qualities (meta-features) of a specified portion of a data stream.</p> <pre><code>    DataQuality qualities = client.dataQuality(1,0,10000,null);\n</code></pre>"},{"location":"ecosystem/Java/#dataqualitylist","title":"<code>dataQualityList()</code>","text":"<p>Retrieves a list of all data qualities known to OpenML.</p> <pre><code>    DataQualityList response = client.dataQualityList();\n    String[] qualities = response.getQualities();\n</code></pre>"},{"location":"ecosystem/Java/#data-upload","title":"Data upload","text":""},{"location":"ecosystem/Java/#datauploaddatasetdescription-description-file-dataset","title":"<code>dataUpload(DataSetDescription description, File dataset)</code>","text":"<p>Uploads a data set file to OpenML given a description. Throws an exception if the upload failed, see openml.data.upload for error codes.</p> <pre><code>    DataSetDescription dataset = new DataSetDescription( \"iris\", \"The iris dataset\", \"arff\", \"class\");\n    UploadDataSet data = client.dataUpload( dataset, new File(\"data/path\"));\n    int data_id = result.getId();\n</code></pre>"},{"location":"ecosystem/Java/#datauploaddatasetdescription-description","title":"<code>dataUpload(DataSetDescription description)</code>","text":"<p>Registers an existing dataset (hosted elsewhere). The description needs to include the url of the data set. Throws an exception if the upload failed, see openml.data.upload for error codes.</p> <pre><code>    DataSetDescription description = new DataSetDescription( \"iris\", \"The iris dataset\", \"arff\", \"class\");\n    description.setUrl(\"http://datarepository.org/mydataset\");\n    UploadDataSet data = client.dataUpload( description );\n    int data_id = result.getId();\n</code></pre>"},{"location":"ecosystem/Java/#flow-download","title":"Flow download","text":""},{"location":"ecosystem/Java/#flowgetint-flow_id","title":"<code>flowGet(int flow_id)</code>","text":"<p>Retrieves the description of the flow/implementation with the given id.</p> <pre><code>    Implementation flow = client.flowGet(100);\n    String name = flow.getName();\n    String version = flow.getVersion();\n    String description = flow.getDescription();\n    String binary_url = flow.getBinary_url();\n    String source_url = flow.getSource_url();\n    Parameter[] parameters = flow.getParameter();\n</code></pre>"},{"location":"ecosystem/Java/#flow-management","title":"Flow management","text":""},{"location":"ecosystem/Java/#flowowned","title":"<code>flowOwned()</code>","text":"<p>Retrieves an array of id's of all flows/implementations owned by you.</p> <pre><code>    ImplementationOwned response = client.flowOwned();\n    Integer[] ids = response.getIds();\n</code></pre>"},{"location":"ecosystem/Java/#flowexistsstring-name-string-version","title":"<code>flowExists(String name, String version)</code>","text":"<p>Checks whether an implementation with the given name and version is already registered on OpenML.</p> <pre><code>    ImplementationExists check = client.flowExists(\"weka.j48\", \"3.7.12\");\n    boolean exists = check.exists();\n    int flow_id = check.getId();\n</code></pre>"},{"location":"ecosystem/Java/#flowdeleteint-id","title":"<code>flowDelete(int id)</code>","text":"<p>Removes the flow with the given id (if you are its owner).</p> <pre><code>    ImplementationDelete response = client.openmlImplementationDelete(100);\n</code></pre>"},{"location":"ecosystem/Java/#flow-upload","title":"Flow upload","text":""},{"location":"ecosystem/Java/#flowuploadimplementation-description-file-binary-file-source","title":"<code>flowUpload(Implementation description, File binary, File source)</code>","text":"<p>Uploads implementation files (binary and/or source) to OpenML given a description.</p> <pre><code>    Implementation flow = new Implementation(\"weka.J48\", \"3.7.12\", \"description\", \"Java\", \"WEKA 3.7.12\")\n    UploadImplementation response = client.flowUpload( flow, new File(\"code.jar\"), new File(\"source.zip\"));\n    int flow_id = response.getId();\n</code></pre>"},{"location":"ecosystem/Java/#task-download","title":"Task download","text":""},{"location":"ecosystem/Java/#taskgetint-task_id","title":"<code>taskGet(int task_id)</code>","text":"<p>Retrieves the description of the task with the given id.</p> <pre><code>    Task task = client.taskGet(1);\n    String task_type = task.getTask_type();\n    Input[] inputs = task.getInputs();\n    Output[] outputs = task.getOutputs();\n</code></pre>"},{"location":"ecosystem/Java/#taskevaluationsint-task_id","title":"<code>taskEvaluations(int task_id)</code>","text":"<p>Retrieves all evaluations for the task with the given id.</p> <pre><code>    TaskEvaluations response = client.taskEvaluations(1);\n    Evaluation[] evaluations = response.getEvaluation();\n</code></pre>"},{"location":"ecosystem/Java/#taskevaluationsint-task_id-int-start-int-end-int-interval_size","title":"<code>taskEvaluations(int task_id, int start, int end, int interval_size)</code>","text":"<p>For data streams. Retrieves all evaluations for the task over the specified window of the stream.</p> <pre><code>    TaskEvaluations response = client.taskEvaluations(1);\n    Evaluation[] evaluations = response.getEvaluation();\n</code></pre>"},{"location":"ecosystem/Java/#run-download","title":"Run download","text":""},{"location":"ecosystem/Java/#rungetint-run_id","title":"<code>runGet(int run_id)</code>","text":"<p>Retrieves the description of the run with the given id.</p> <pre><code>    Run run = client.runGet(1);\n    int task_id = run.getTask_id();\n    int flow_id = run.getImplementation_id();\n    Parameter_setting[] settings = run.getParameter_settings()\n    EvaluationScore[] scores = run.getOutputEvaluation();\n</code></pre>"},{"location":"ecosystem/Java/#run-management","title":"Run management","text":""},{"location":"ecosystem/Java/#rundeleteint-run_id","title":"<code>runDelete(int run_id)</code>","text":"<p>Deletes the run with the given id (if you are its owner).</p> <pre><code>    RunDelete response = client.runDelete(1);\n</code></pre>"},{"location":"ecosystem/Java/#run-upload","title":"Run upload","text":""},{"location":"ecosystem/Java/#runuploadrun-description-mapstringfile-output_files","title":"<code>runUpload(Run description, Map&lt;String,File&gt; output_files)</code>","text":"<p>Uploads a run to OpenML, including a description and a set of output files depending on the task type.</p> <pre><code>    Run.Parameter_setting[] parameter_settings = new Run.Parameter_setting[1];\n    parameter_settings[0] = Run.Parameter_setting(null, \"M\", \"2\");\n    Run run = new Run(\"1\", null, \"100\", \"setup_string\", parameter_settings);\n    Map outputs = new HashMap&lt;String,File&gt;();\n    outputs.add(\"predictions\",new File(\"predictions.arff\"));\n    UploadRun response = client.runUpload( run, outputs);\n    int run_id = response.getRun_id();\n</code></pre>"},{"location":"ecosystem/MOA/","title":"MOA","text":"<p>OpenML features extensive support for MOA. However currently this is implemented as a stand alone MOA compilation, using the latest version (as of May, 2014).</p> <p>Download MOA for OpenML</p>"},{"location":"ecosystem/MOA/#quick-start","title":"Quick Start","text":"<ul> <li>Download the standalone MOA environment above.</li> <li>Find your API key in your profile (log in first). Create a config file called <code>openml.conf</code> in a <code>.openml</code> directory in your home dir. It should contain the following lines: <p>api_key = YOUR_KEY</p> </li> <li>Launch the JAR file by double clicking on it, or launch from command-line using the following command: <p>java -cp openmlmoa.beta.jar moa.gui.GUI</p> </li> <li>Select the task <code>moa.tasks.openml.OpenmlDataStreamClassification</code> to evaluate a classifier on an OpenML task, and send the results to OpenML.</li> <li>Optionally, you can generate new streams using the Bayesian Network Generator: select the <code>moa.tasks.WriteStreamToArff</code> task, with <code>moa.streams.generators.BayesianNetworkGenerator</code>.</li> </ul>"},{"location":"ecosystem/Python_extensions/","title":"Integrating your Python libraries","text":"<p>OpenML-Python provides an extension interface to connect other machine learning libraries than scikit-learn to OpenML. Please check the <code>api_extensions</code> and use the scikit-learn extension in <code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text role=\"class\"} as a starting point.</p>"},{"location":"ecosystem/Python_extensions/#connecting-new-machine-learning-libraries","title":"Connecting new machine learning libraries","text":""},{"location":"ecosystem/Python_extensions/#content-of-the-library","title":"Content of the Library","text":"<p>To leverage support from the community and to tap in the potential of OpenML, interfacing with popular machine learning libraries is essential. The OpenML-Python package is capable of downloading meta-data and results (data, flows, runs), regardless of the library that was used to upload it. However, in order to simplify the process of uploading flows and runs from a specific library, an additional interface can be built. The OpenML-Python team does not have the capacity to develop and maintain such interfaces on its own. For this reason, we have built an extension interface to allows others to contribute back. Building a suitable extension for therefore requires an understanding of the current OpenML-Python support.</p> <p>The <code>sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py</code>{.interpreted-text role=\"ref\"} tutorial shows how scikit-learn currently works with OpenML-Python as an extension. The sklearn extension packaged with the openml-python repository can be used as a template/benchmark to build the new extension.</p>"},{"location":"ecosystem/Python_extensions/#api","title":"API","text":"<ul> <li>The extension scripts must import the [openml]{.title-ref} package     and be able to interface with any function from the OpenML-Python     <code>api</code>.</li> <li>The extension has to be defined as a Python class and must inherit     from <code>openml.extensions.Extension</code>.</li> <li>This class needs to have all the functions from [class     Extension]{.title-ref} overloaded as required.</li> <li>The redefined functions should have adequate and appropriate     docstrings. The [Sklearn Extension API     :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref}     is a good example to follow.</li> </ul>"},{"location":"ecosystem/Python_extensions/#interfacing-with-openml-python","title":"Interfacing with OpenML-Python","text":"<p>Once the new extension class has been defined, the openml-python module to <code>openml.extensions.register_extension</code> must be called to allow OpenML-Python to interface the new extension.</p> <p>The following methods should get implemented. Although the documentation in the [Extension]{.title-ref} interface should always be leading, here we list some additional information and best practices. The [Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is a good example to follow. Note that most methods are relatively simple and can be implemented in several lines of code.</p> <ul> <li>General setup (required)<ul> <li><code>can_handle_flow</code>: Takes as     argument an OpenML flow, and checks whether this can be handled     by the current extension. The OpenML database consists of many     flows, from various workbenches (e.g., scikit-learn, Weka, mlr).     This method is called before a model is being deserialized.     Typically, the flow-dependency field is used to check whether     the specific library is present, and no unknown libraries are     present there.</li> <li><code>can_handle_model</code>: Similar as     <code>can_handle_flow</code>, except that in     this case a Python object is given. As such, in many cases, this     method can be implemented by checking whether this adheres to a     certain base class.</li> </ul> </li> <li>Serialization and De-serialization (required)<ul> <li><code>flow_to_model</code>: deserializes the     OpenML Flow into a model (if the library can indeed handle the     flow). This method has an important interplay with     <code>model_to_flow</code>. Running these     two methods in succession should result in exactly the same     model (or flow). This property can be used for unit testing     (e.g., build a model with hyperparameters, make predictions on a     task, serialize it to a flow, deserialize it back, make it     predict on the same task, and check whether the predictions are     exactly the same.) The example in the scikit-learn interface     might seem daunting, but note that here some complicated design     choices were made, that allow for all sorts of interesting     research questions. It is probably good practice to start easy.</li> <li><code>model_to_flow</code>: The inverse of     <code>flow_to_model</code>. Serializes a     model into an OpenML Flow. The flow should preserve the class,     the library version, and the tunable hyperparameters.</li> <li><code>get_version_information</code>: Return     a tuple with the version information of the important libraries.</li> <li><code>create_setup_string</code>: No longer     used, and will be deprecated soon.</li> </ul> </li> <li>Performing runs (required)<ul> <li><code>is_estimator</code>: Gets as input a     class, and checks whether it has the status of estimator in the     library (typically, whether it has a train method and a predict     method).</li> <li><code>seed_model</code>: Sets a random seed     to the model.</li> <li><code>_run_model_on_fold</code>: One of the     main requirements for a library to generate run objects for the     OpenML server. Obtains a train split (with labels) and a test     split (without labels) and the goal is to train a model on the     train split and return the predictions on the test split. On top     of the actual predictions, also the class probabilities should     be determined. For classifiers that do not return class     probabilities, this can just be the hot-encoded predicted label.     The predictions will be evaluated on the OpenML server. Also,     additional information can be returned, for example,     user-defined measures (such as runtime information, as this can     not be inferred on the server). Additionally, information about     a hyperparameter optimization trace can be provided.</li> <li><code>obtain_parameter_values</code>:     Obtains the hyperparameters of a given model and the current     values. Please note that in the case of a hyperparameter     optimization procedure (e.g., random search), you only should     return the hyperparameters of this procedure (e.g., the     hyperparameter grid, budget, etc) and that the chosen model will     be inferred from the optimization trace.</li> <li><code>check_if_model_fitted</code>: Check     whether the train method of the model has been called (and as     such, whether the predict method can be used).</li> </ul> </li> <li>Hyperparameter optimization (optional)<ul> <li><code>instantiate_model_from_hpo_class</code>{.interpreted-text     role=\"meth\"}: If a given run has recorded the hyperparameter     optimization trace, then this method can be used to     reinstantiate the model with hyperparameters of a given     hyperparameter optimization iteration. Has some similarities     with <code>flow_to_model</code> (as this     method also sets the hyperparameters of a model). Note that     although this method is required, it is not necessary to     implement any logic if hyperparameter optimization is not     implemented. Simply raise a [NotImplementedError]{.title-ref}     then.</li> </ul> </li> </ul>"},{"location":"ecosystem/Python_extensions/#hosting-the-library","title":"Hosting the library","text":"<p>Each extension created should be a stand-alone repository, compatible with the OpenML-Python repository. The extension repository should work off-the-shelf with OpenML-Python installed.</p> <p>Create a public Github repo with the following directory structure:</p> <pre><code>| [repo name]\n|    |-- [extension name]\n|    |    |-- __init__.py\n|    |    |-- extension.py\n|    |    |-- config.py (optionally)\n</code></pre>"},{"location":"ecosystem/Python_extensions/#recommended","title":"Recommended","text":"<ul> <li>Test cases to keep the extension up to date with the     [openml-python]{.title-ref} upstream changes.</li> <li>Documentation of the extension API, especially if any new     functionality added to OpenML-Python\\'s extension design.</li> <li>Examples to show how the new extension interfaces and works with     OpenML-Python.</li> <li>Create a PR to add the new extension to the OpenML-Python API     documentation.</li> </ul> <p>Happy contributing!</p>"},{"location":"ecosystem/Rest/","title":"REST tutorial","text":"<p>OpenML offers a RESTful Web API, with predictive URLs, for uploading and downloading machine learning resources. Try the API Documentation to see examples of all calls, and test them right in your browser.</p>"},{"location":"ecosystem/Rest/#getting-started","title":"Getting started","text":"<ul> <li>REST services can be called using simple HTTP GET or POST actions.</li> <li>The REST Endpoint URL is <code>https://www.openml.org/api/v1/</code></li> <li>The default endpoint returns data in XML. If you prefer JSON, use the endpoint <code>https://www.openml.org/api/v1/json/</code>. Note that, to upload content, you still need to use XML (at least for now).</li> </ul>"},{"location":"ecosystem/Rest/#testing","title":"Testing","text":"<p>For continuous integration and testing purposes, we have a test server offering the same API, but which does not affect the production server.</p> <ul> <li>The test server REST Endpoint URL is <code>https://test.openml.org/api/v1/</code></li> </ul>"},{"location":"ecosystem/Rest/#error-messages","title":"Error messages","text":"<p>Error messages will look like this:</p> <pre><code>&lt;oml:error xmlns:oml=\"http://openml.org/error\"&gt;\n&lt;oml:code&gt;100&lt;/oml:code&gt;\n&lt;oml:message&gt;Please invoke legal function&lt;/oml:message&gt;\n&lt;oml:additional_information&gt;Additional information, not always available.&lt;/oml:additional_information&gt;\n&lt;/oml:error&gt;\n</code></pre> <p>All error messages are listed in the API documentation. E.g. try to get a non-existing dataset:</p> <ul> <li>in XML: https://www.openml.org/api_new/v1/data/99999</li> <li>in JSON: https://www.openml.org/api_new/v1/json/data/99999</li> </ul>"},{"location":"ecosystem/Rest/#examples","title":"Examples","text":"<p>You need to be logged in for these examples to work.</p>"},{"location":"ecosystem/Rest/#download-a-dataset","title":"Download a dataset","text":"<ul> <li>User asks for a dataset using the /data/{id} service. The <code>dataset id</code> is typically part of a task, or can be found on OpenML.org.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting the dataset. This can be OpenML, but also any other data repository. Try it now</li> </ul>"},{"location":"ecosystem/Rest/#download-a-flow","title":"Download a flow","text":"<ul> <li>User asks for a flow using the /flow/{id} service and a <code>flow id</code>. The <code>flow id</code> can be found on OpenML.org.</li> <li>OpenML returns a description of the flow as an XML file (or JSON). Try it now</li> <li>The flow description contains the URL where the flow can be downloaded (e.g. GitHub), either as source, binary or both, as well as additional information on history, dependencies and licence. The user calls the right URL to download it.</li> <li>The flow is returned by the server hosting it. This can be OpenML, but also any other code repository. Try it now</li> </ul>"},{"location":"ecosystem/Rest/#download-a-task","title":"Download a task","text":"<ul> <li>User asks for a task using the /task/{id} service and a <code>task id</code>. The <code>task id</code> is typically returned when searching for tasks.</li> <li>OpenML returns a description of the task as an XML file (or JSON). Try it now</li> <li>The task description contains the <code>dataset id</code>(s) of the datasets involved in this task. The user asks for the dataset using the /data/{id} service and the <code>dataset id</code>.</li> <li>OpenML returns a description of the dataset as an XML file (or JSON). Try it now</li> <li>The dataset description contains the URL where the dataset can be downloaded. The user calls that URL to download the dataset.</li> <li>The dataset is returned by the server hosting it. This can be OpenML, but also any other data repository. Try it now</li> <li>The task description may also contain links to other resources, such as the train-test splits to be used in cross-validation. The user calls that URL to download the train-test splits.</li> <li>The train-test splits are returned by OpenML. Try it now</li> </ul>"},{"location":"ecosystem/Weka/","title":"Weka","text":"<p>OpenML is integrated in the Weka (Waikato Environment for Knowledge Analysis) Experimenter and the Command Line Interface.</p>"},{"location":"ecosystem/Weka/#installation","title":"Installation","text":"<p>OpenML is available as a weka extension in the package manager:</p> <ul> <li>Download the latest version (3.7.13 or higher).</li> <li>Launch Weka, or start from commandline: <p>java -jar weka.jar</p> </li> <li>If you need more memory (e.g. 1GB), start as follows: <p>java -Xmx1G -jar weka.jar</p> </li> <li>Open the package manager (Under 'Tools')</li> <li>Select package OpenmlWeka and click install. Afterwards, restart WEKA.</li> <li>From the Tools menu, open the 'OpenML Experimenter'.</li> </ul>"},{"location":"ecosystem/Weka/#graphical-interface","title":"Graphical Interface","text":"<p>You can solve OpenML Tasks in the Weka Experimenter, and automatically upload your experiments to OpenML (or store them locally).  </p> <ul> <li>From the Tools menu, open the 'OpenML Experimenter'.</li> <li>Enter your API key in the top field (log in first). You can also store this in a config file (see below).</li> <li>In the 'Tasks' panel, click the 'Add New' button to add new tasks. Insert the task id's as comma-separated values (e.g., '1,2,3,4,5'). Use the search function on OpenML to find interesting tasks and click the ID icon to list the ID's. In the future this search will also be integrated in WEKA.</li> <li>Add algorithms in the \"Algorithm\" panel.</li> <li>Go to the \"Run\" tab, and click on the \"Start\" button.</li> <li>The experiment will be executed and sent to OpenML.org.</li> <li>The runs will now appear on OpenML.org. You can follow their progress and check for errors on your profile page under 'Runs'.</li> </ul>"},{"location":"ecosystem/Weka/#commandline-interface","title":"CommandLine Interface","text":"<p>The Command Line interface is useful for running experiments automatically on a server, without using a GUI.</p> <ul> <li>Create a config file called <code>openml.conf</code> in a new directory called <code>.openml</code> in your home dir. It should contain the following line: <p>api_key = YOUR_KEY</p> </li> <li>Execute the following command: <p>java -cp weka.jar openml.experiment.TaskBasedExperiment -T  -C  --  <li>For example, the following command will run Weka's J48 algorithm on Task 1: <p>java -cp OpenWeka.beta.jar openml.experiment.TaskBasedExperiment -T 1 -C weka.classifiers.trees.J48</p> </li> <li>The following suffix will set some parameters of this classifier: <p>-- -C 0.25 -M 2</p> </li>"},{"location":"ecosystem/Weka/#api-reference","title":"API reference","text":"<p>Check the Weka integration Java Docs for more details about the possibilities.</p>"},{"location":"ecosystem/Weka/#issues","title":"Issues","text":"<p>Please report any bugs that you may encounter in the issue tracker: https://github.com/openml/openml-weka Or email to j.n.van.rijn@liacs.leidenuniv.nl</p>"},{"location":"ecosystem/mlr/","title":"Machine Learning in R (mlr)","text":"<p>OpenML is readily integrated with mlr through the mlr3oml package.</p> <p>Example</p> <pre><code>library(mlr3oml)\nlibrary(mlr3)\n\n# Search for specific datasets\nodatasets = list_oml_data(\nnumber_features = c(10, 20),\nnumber_instances = c(45000, 50000),\nnumber_classes = 2\n)\n\n# Get dataset\nodata = odt(id = 1590)\n# Access the actual data\nodata$data\n\n# Convert to an mlr3::Task\ntsk_adult = as_task(odata, target = \"class\")\n</code></pre> <p>Key features:  </p> <ul> <li>Query and download OpenML datasets and use them however you like  </li> <li>Build any mlr learner, run it on any task and save the experiment as run objects  </li> <li>Upload your runs for collaboration or publishing  </li> <li>Query, download and reuse all shared runs  </li> </ul> <p>There is also an older (deprecated) OpenML R package.</p>"},{"location":"ecosystem/Scikit-learn/","title":"scikit-learn","text":"<p>OpenML is readily integrated with scikit-learn through the Python API. This page provides a brief overview of the key features and installation instructions. For more detailed API documentation, please refer to the official documentation.</p>"},{"location":"ecosystem/Scikit-learn/#key-features","title":"Key features:","text":"<ul> <li>Query and download OpenML datasets and use them however you like</li> <li>Build any sklearn estimator or pipeline and convert to OpenML flows</li> <li>Run any flow on any task and save the experiment as run objects</li> <li>Upload your runs for collaboration or publishing</li> <li>Query, download and reuse all shared runs</li> </ul>"},{"location":"ecosystem/Scikit-learn/#installation","title":"Installation","text":"<pre><code>pip install openml\n</code></pre>"},{"location":"ecosystem/Scikit-learn/#query-and-download-data","title":"Query and download data","text":"<pre><code>import openml\n\n# List all datasets and their properties\nopenml.datasets.list_datasets(output_format=\"dataframe\")\n\n# Get dataset by ID\ndataset = openml.datasets.get_dataset(61)\n\n# Get dataset by name\ndataset = openml.datasets.get_dataset('Fashion-MNIST')\n\n# Get the data itself as a dataframe (or otherwise)\nX, y, _, _ = dataset.get_data(dataset_format=\"dataframe\")\n</code></pre>"},{"location":"ecosystem/Scikit-learn/#download-tasks-run-models-locally-publish-results-with-scikit-learn","title":"Download tasks, run models locally, publish results (with scikit-learn)","text":"<pre><code>from sklearn import ensemble\nfrom openml import tasks, runs\n\n# Build any model you like\nclf = ensemble.RandomForestClassifier()\n\n# Download any OpenML task\ntask = tasks.get_task(3954)\n\n# Run and evaluate your model on the task\nrun = runs.run_model_on_task(clf, task)\n\n# Share the results on OpenML. Your API key can be found in your account.\n# openml.config.apikey = 'YOUR_KEY'\nrun.publish()\n</code></pre>"},{"location":"ecosystem/Scikit-learn/#openml-benchmarks","title":"OpenML Benchmarks","text":"<pre><code># List all tasks in a benchmark\nbenchmark = openml.study.get_suite('OpenML-CC18')\ntasks.list_tasks(output_format=\"dataframe\", task_id=benchmark.tasks)\n\n# Return benchmark results\nopenml.evaluations.list_evaluations(\n    function=\"area_under_roc_curve\",\n    tasks=benchmark.tasks,\n    output_format=\"dataframe\"\n)\n</code></pre>"},{"location":"ecosystem/Scikit-learn/basic_tutorial/","title":"Basic tutorial","text":"In\u00a0[12]: Copied! <pre>from IPython.display import display, HTML, Markdown\nimport os\nimport yaml\nwith open(\"../../../mkdocs.yml\", \"r\") as f:\n    load_config = yaml.safe_load(f)\nrepo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\")\nbinder_url = load_config[\"binder_url\"]\nrelative_file_path = \"integrations/Scikit-learn/basic_tutorial.ipynb\"\ndisplay(HTML(f\"\"\"&lt;a target=\"_blank\" href=\"https://colab.research.google.com/github/{repo_url}/{relative_file_path}\"&gt;\n  &lt;img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/&gt;\n&lt;/a&gt;\"\"\"))\ndisplay(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\"))\n</pre> from IPython.display import display, HTML, Markdown import os import yaml with open(\"../../../mkdocs.yml\", \"r\") as f:     load_config = yaml.safe_load(f) repo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\") binder_url = load_config[\"binder_url\"] relative_file_path = \"integrations/Scikit-learn/basic_tutorial.ipynb\" display(HTML(f\"\"\" \"\"\")) display(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\")) In\u00a0[\u00a0]: Copied! <pre>!pip install openml\n</pre> !pip install openml In\u00a0[2]: Copied! <pre>import openml\nfrom sklearn import impute, tree, pipeline\n</pre> import openml from sklearn import impute, tree, pipeline In\u00a0[7]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <pre>/Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!\n  warnings.warn(\n</pre> In\u00a0[8]: Copied! <pre># Define a scikit-learn classifier or pipeline\nclf = pipeline.Pipeline(\n    steps=[\n        ('imputer', impute.SimpleImputer()),\n        ('estimator', tree.DecisionTreeClassifier())\n    ]\n)\n</pre>  # Define a scikit-learn classifier or pipeline clf = pipeline.Pipeline(     steps=[         ('imputer', impute.SimpleImputer()),         ('estimator', tree.DecisionTreeClassifier())     ] )  In\u00a0[9]: Copied! <pre># Download the OpenML task for the pendigits dataset with 10-fold\n# cross-validation.\ntask = openml.tasks.get_task(32)\ntask\n</pre>  # Download the OpenML task for the pendigits dataset with 10-fold # cross-validation. task = openml.tasks.get_task(32) task Out[9]: <pre>OpenML Classification Task\n==========================\nTask Type Description: https://test.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION\nTask ID..............: 32\nTask URL.............: https://test.openml.org/t/32\nEstimation Procedure.: crossvalidation\nTarget Feature.......: class\n# of Classes.........: 10\nCost Matrix..........: Available</pre> In\u00a0[11]: Copied! <pre># Run the scikit-learn model on the task.\nrun = openml.runs.run_model_on_task(clf, task)\n# Publish the experiment on OpenML (optional, requires an API key.\n# You can get your own API key by signing up to OpenML.org)\n</pre> # Run the scikit-learn model on the task. run = openml.runs.run_model_on_task(clf, task) # Publish the experiment on OpenML (optional, requires an API key. # You can get your own API key by signing up to OpenML.org)  In\u00a0[\u00a0]: Copied! <pre>run.publish()\nprint(f'View the run online: {run.openml_url}')\n</pre>  run.publish() print(f'View the run online: {run.openml_url}')"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/","title":"Datasets","text":"In\u00a0[2]: Copied! <pre>from IPython.display import display, HTML, Markdown\nimport os\nimport yaml\nwith open(\"../../../mkdocs.yml\", \"r\") as f:\n    load_config = yaml.safe_load(f)\nrepo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\")\nbinder_url = load_config[\"binder_url\"]\nrelative_file_path = \"integrations/Scikit-learn/datasets_tutorial.ipynb\"\ndisplay(HTML(f\"\"\"&lt;a target=\"_blank\" href=\"https://colab.research.google.com/github/{repo_url}/{relative_file_path}\"&gt;\n  &lt;img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/&gt;\n&lt;/a&gt;\"\"\"))\ndisplay(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\"))\n</pre> from IPython.display import display, HTML, Markdown import os import yaml with open(\"../../../mkdocs.yml\", \"r\") as f:     load_config = yaml.safe_load(f) repo_url = load_config[\"repo_url\"].replace(\"https://github.com/\", \"\") binder_url = load_config[\"binder_url\"] relative_file_path = \"integrations/Scikit-learn/datasets_tutorial.ipynb\" display(HTML(f\"\"\" \"\"\")) display(Markdown(\"[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/SubhadityaMukherjee/openml_docs/HEAD?labpath=Scikit-learn%2Fdatasets_tutorial)\")) In\u00a0[9]: Copied! <pre>!pip install openml\n</pre> !pip install openml <pre>Requirement already satisfied: openml in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (0.14.2)\nRequirement already satisfied: scikit-learn&gt;=0.18 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (1.4.2)\nRequirement already satisfied: requests in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.31.0)\nRequirement already satisfied: liac-arff&gt;=2.4.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.5.0)\nRequirement already satisfied: numpy&gt;=1.6.2 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (1.26.4)\nRequirement already satisfied: minio in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (7.2.7)\nRequirement already satisfied: pandas&gt;=1.0.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.2.2)\nRequirement already satisfied: scipy&gt;=0.13.3 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (1.13.0)\nRequirement already satisfied: pyarrow in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (16.0.0)\nRequirement already satisfied: xmltodict in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (0.13.0)\nRequirement already satisfied: python-dateutil in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from openml) (2.9.0.post0)\nRequirement already satisfied: tzdata&gt;=2022.7 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from pandas&gt;=1.0.0-&gt;openml) (2024.1)\nRequirement already satisfied: pytz&gt;=2020.1 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from pandas&gt;=1.0.0-&gt;openml) (2024.1)\nRequirement already satisfied: six&gt;=1.5 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from python-dateutil-&gt;openml) (1.16.0)\nRequirement already satisfied: threadpoolctl&gt;=2.0.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from scikit-learn&gt;=0.18-&gt;openml) (3.5.0)\nRequirement already satisfied: joblib&gt;=1.2.0 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from scikit-learn&gt;=0.18-&gt;openml) (1.4.0)\nRequirement already satisfied: urllib3 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (2.2.1)\nRequirement already satisfied: typing-extensions in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (4.11.0)\nRequirement already satisfied: pycryptodome in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (3.20.0)\nRequirement already satisfied: certifi in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (2024.2.2)\nRequirement already satisfied: argon2-cffi in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from minio-&gt;openml) (23.1.0)\nRequirement already satisfied: idna&lt;4,&gt;=2.5 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from requests-&gt;openml) (3.7)\nRequirement already satisfied: charset-normalizer&lt;4,&gt;=2 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from requests-&gt;openml) (3.3.2)\nRequirement already satisfied: argon2-cffi-bindings in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from argon2-cffi-&gt;minio-&gt;openml) (21.2.0)\nRequirement already satisfied: cffi&gt;=1.0.1 in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from argon2-cffi-bindings-&gt;argon2-cffi-&gt;minio-&gt;openml) (1.16.0)\nRequirement already satisfied: pycparser in /Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages (from cffi&gt;=1.0.1-&gt;argon2-cffi-bindings-&gt;argon2-cffi-&gt;minio-&gt;openml) (2.22)\n\n[notice] A new release of pip is available: 23.0.1 -&gt; 24.0\n[notice] To update, run: pip install --upgrade pip\n</pre> In\u00a0[2]: Copied! <pre># License: BSD 3-Clauses\n\nimport openml\nimport pandas as pd\nfrom openml.datasets import edit_dataset, fork_dataset, get_dataset\n</pre> # License: BSD 3-Clauses  import openml import pandas as pd from openml.datasets import edit_dataset, fork_dataset, get_dataset In\u00a0[3]: Copied! <pre>datalist = openml.datasets.list_datasets(output_format=\"dataframe\")\ndatalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]\n\nprint(f\"First 10 of {len(datalist)} datasets...\")\ndatalist.head(n=10)\n\n# The same can be done with lesser lines of code\nopenml_df = openml.datasets.list_datasets(output_format=\"dataframe\")\nopenml_df.head(n=10)\n</pre> datalist = openml.datasets.list_datasets(output_format=\"dataframe\") datalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]  print(f\"First 10 of {len(datalist)} datasets...\") datalist.head(n=10)  # The same can be done with lesser lines of code openml_df = openml.datasets.list_datasets(output_format=\"dataframe\") openml_df.head(n=10) <pre>First 10 of 5466 datasets...\n</pre> Out[3]: did name version uploader status format MajorityClassSize MaxNominalAttDistinctValues MinorityClassSize NumberOfClasses NumberOfFeatures NumberOfInstances NumberOfInstancesWithMissingValues NumberOfMissingValues NumberOfNumericFeatures NumberOfSymbolicFeatures 2 2 anneal 1 1 active ARFF 684.0 7.0 8.0 5.0 39.0 898.0 898.0 22175.0 6.0 33.0 3 3 kr-vs-kp 1 1 active ARFF 1669.0 3.0 1527.0 2.0 37.0 3196.0 0.0 0.0 0.0 37.0 4 4 labor 1 1 active ARFF 37.0 3.0 20.0 2.0 17.0 57.0 56.0 326.0 8.0 9.0 5 5 arrhythmia 1 1 active ARFF 245.0 13.0 2.0 13.0 280.0 452.0 384.0 408.0 206.0 74.0 6 6 letter 1 1 active ARFF 813.0 26.0 734.0 26.0 17.0 20000.0 0.0 0.0 16.0 1.0 7 7 audiology 1 1 active ARFF 57.0 24.0 1.0 24.0 70.0 226.0 222.0 317.0 0.0 70.0 8 8 liver-disorders 1 1 active ARFF NaN NaN NaN 0.0 6.0 345.0 0.0 0.0 6.0 0.0 9 9 autos 1 1 active ARFF 67.0 22.0 3.0 6.0 26.0 205.0 46.0 59.0 15.0 11.0 10 10 lymph 1 1 active ARFF 81.0 8.0 2.0 4.0 19.0 148.0 0.0 0.0 3.0 16.0 11 11 balance-scale 1 1 active ARFF 288.0 3.0 49.0 3.0 5.0 625.0 0.0 0.0 4.0 1.0 In\u00a0[4]: Copied! <pre>datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20)\n\"\"\ndatalist.query('name == \"eeg-eye-state\"')\n\"\"\ndatalist.query(\"NumberOfClasses &gt; 50\")\n</pre> datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20) \"\" datalist.query('name == \"eeg-eye-state\"') \"\" datalist.query(\"NumberOfClasses &gt; 50\") Out[4]: did name NumberOfInstances NumberOfFeatures NumberOfClasses 1491 1491 one-hundred-plants-margin 1600.0 65.0 100.0 1492 1492 one-hundred-plants-shape 1600.0 65.0 100.0 1493 1493 one-hundred-plants-texture 1599.0 65.0 100.0 4552 4552 BachChoralHarmony 5665.0 17.0 102.0 41167 41167 dionis 416188.0 61.0 355.0 41169 41169 helena 65196.0 28.0 100.0 41960 41960 seattlecrime6 523590.0 8.0 144.0 41983 41983 CIFAR-100 60000.0 3073.0 100.0 42078 42078 beer_reviews 1586614.0 13.0 104.0 42087 42087 beer_reviews 1586614.0 13.0 104.0 42088 42088 beer_reviews 1586614.0 13.0 104.0 42089 42089 vancouver_employee 1586614.0 13.0 104.0 42123 42123 article_influence 3615.0 7.0 3169.0 42223 42223 dataset-autoHorse_fixed 201.0 69.0 186.0 42396 42396 aloi 108000.0 129.0 1000.0 43723 43723 Toronto-Apartment-Rental-Price 1124.0 7.0 188.0 44282 44282 Meta_Album_PLK_Mini 3440.0 3.0 86.0 44283 44283 Meta_Album_FLW_Mini 4080.0 3.0 102.0 44284 44284 Meta_Album_SPT_Mini 2920.0 3.0 73.0 44285 44285 Meta_Album_BRD_Mini 12600.0 3.0 315.0 44288 44288 Meta_Album_TEX_Mini 2560.0 3.0 64.0 44289 44289 Meta_Album_CRS_Mini 7840.0 3.0 196.0 44292 44292 Meta_Album_INS_2_Mini 4080.0 3.0 102.0 44298 44298 Meta_Album_DOG_Mini 4800.0 3.0 120.0 44304 44304 Meta_Album_TEX_ALOT_Mini 10000.0 3.0 250.0 44306 44306 Meta_Album_INS_Mini 4160.0 3.0 104.0 44317 44317 Meta_Album_PLK_Extended 473273.0 3.0 102.0 44318 44318 Meta_Album_FLW_Extended 8189.0 3.0 102.0 44319 44319 Meta_Album_SPT_Extended 10416.0 3.0 73.0 44320 44320 Meta_Album_BRD_Extended 49054.0 3.0 315.0 44322 44322 Meta_Album_TEX_Extended 8675.0 3.0 64.0 44323 44323 Meta_Album_CRS_Extended 16185.0 3.0 196.0 44326 44326 Meta_Album_INS_2_Extended 75222.0 3.0 102.0 44331 44331 Meta_Album_DOG_Extended 20480.0 3.0 120.0 44337 44337 Meta_Album_TEX_ALOT_Extended 25000.0 3.0 250.0 44340 44340 Meta_Album_INS_Extended 170506.0 3.0 117.0 44533 44533 dionis_seed_0_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44534 44534 dionis_seed_1_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44535 44535 dionis_seed_2_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44536 44536 dionis_seed_3_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44537 44537 dionis_seed_4_nrows_2000_nclasses_10_ncols_100... 2000.0 61.0 355.0 44728 44728 helena_seed_0_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44729 44729 helena_seed_1_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44730 44730 helena_seed_2_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44731 44731 helena_seed_3_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 44732 44732 helena_seed_4_nrows_2000_nclasses_10_ncols_100... 2000.0 28.0 100.0 45049 45049 MD_MIX_Mini_Copy 28240.0 69.0 706.0 45102 45102 dailybike 731.0 13.0 606.0 45103 45103 dailybike 731.0 13.0 606.0 45104 45104 PLK_Mini_Copy 3440.0 3.0 86.0 45274 45274 PASS 1439588.0 7.0 94137.0 45569 45569 DBLP-QuAD 10000.0 10.0 9999.0 45923 45923 IndoorScenes 15620.0 3.0 67.0 45936 45936 IndoorScenes 15620.0 3.0 67.0 In\u00a0[5]: Copied! <pre># This is done based on the dataset ID.\ndataset = openml.datasets.get_dataset(1471)\n\n# Print a summary\nprint(\n    f\"This is dataset '{dataset.name}', the target feature is \"\n    f\"'{dataset.default_target_attribute}'\"\n)\nprint(f\"URL: {dataset.url}\")\nprint(dataset.description[:500])\n</pre> # This is done based on the dataset ID. dataset = openml.datasets.get_dataset(1471)  # Print a summary print(     f\"This is dataset '{dataset.name}', the target feature is \"     f\"'{dataset.default_target_attribute}'\" ) print(f\"URL: {dataset.url}\") print(dataset.description[:500]) <pre>This is dataset 'eeg-eye-state', the target feature is 'Class'\nURL: https://api.openml.org/data/v1/download/1587924/eeg-eye-state.arff\n**Author**: Oliver Roesler  \n**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/EEG+Eye+State), Baden-Wuerttemberg, Cooperative State University (DHBW), Stuttgart, Germany  \n**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)  \n\nAll data is from one continuous EEG measurement with the Emotiv EEG Neuroheadset. The duration of the measurement was 117 seconds. The eye state was detected via a camera during the EEG measurement and added later manually to the file after\n</pre> <p>Get the actual data.</p> <p>openml-python returns data as pandas dataframes (stored in the <code>eeg</code> variable below), and also some additional metadata that we don't care about right now.</p> In\u00a0[6]: Copied! <pre>eeg, *_ = dataset.get_data()\n</pre> eeg, *_ = dataset.get_data() <p>You can optionally choose to have openml separate out a column from the dataset. In particular, many datasets for supervised problems have a set <code>default_target_attribute</code> which may help identify the target variable.</p> In\u00a0[7]: Copied! <pre>X, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nprint(X.head())\nprint(X.info())\n</pre> X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) print(X.head()) print(X.info()) <pre>        V1       V2       V3       V4       V5       V6       V7       V8  \\\n0  4329.23  4009.23  4289.23  4148.21  4350.26  4586.15  4096.92  4641.03   \n1  4324.62  4004.62  4293.85  4148.72  4342.05  4586.67  4097.44  4638.97   \n2  4327.69  4006.67  4295.38  4156.41  4336.92  4583.59  4096.92  4630.26   \n3  4328.72  4011.79  4296.41  4155.90  4343.59  4582.56  4097.44  4630.77   \n4  4326.15  4011.79  4292.31  4151.28  4347.69  4586.67  4095.90  4627.69   \n\n        V9      V10      V11      V12      V13      V14  \n0  4222.05  4238.46  4211.28  4280.51  4635.90  4393.85  \n1  4210.77  4226.67  4207.69  4279.49  4632.82  4384.10  \n2  4207.69  4222.05  4206.67  4282.05  4628.72  4389.23  \n3  4217.44  4235.38  4210.77  4287.69  4632.31  4396.41  \n4  4210.77  4244.10  4212.82  4288.21  4632.82  4398.46  \n&lt;class 'pandas.core.frame.DataFrame'&gt;\nRangeIndex: 14980 entries, 0 to 14979\nData columns (total 14 columns):\n #   Column  Non-Null Count  Dtype  \n---  ------  --------------  -----  \n 0   V1      14980 non-null  float64\n 1   V2      14980 non-null  float64\n 2   V3      14980 non-null  float64\n 3   V4      14980 non-null  float64\n 4   V5      14980 non-null  float64\n 5   V6      14980 non-null  float64\n 6   V7      14980 non-null  float64\n 7   V8      14980 non-null  float64\n 8   V9      14980 non-null  float64\n 9   V10     14980 non-null  float64\n 10  V11     14980 non-null  float64\n 11  V12     14980 non-null  float64\n 12  V13     14980 non-null  float64\n 13  V14     14980 non-null  float64\ndtypes: float64(14)\nmemory usage: 1.6 MB\nNone\n</pre> <p>Sometimes you only need access to a dataset's metadata. In those cases, you can download the dataset without downloading the data file. The dataset object can be used as normal. Whenever you use any functionality that requires the data, such as <code>get_data</code>, the data will be downloaded. Starting from 0.15, not downloading data will be the default behavior instead. The data will be downloading automatically when you try to access it through openml objects, e.g., using <code>dataset.features</code>.</p> In\u00a0[8]: Copied! <pre>dataset = openml.datasets.get_dataset(1471, download_data=False)\n</pre> dataset = openml.datasets.get_dataset(1471, download_data=False) In\u00a0[9]: Copied! <pre>eegs = eeg.sample(n=1000)\n_ = pd.plotting.scatter_matrix(\n    X.iloc[:100, :4],\n    c=y[:100],\n    figsize=(10, 10),\n    marker=\"o\",\n    hist_kwds={\"bins\": 20},\n    alpha=0.8,\n    cmap=\"plasma\",\n)\n</pre> eegs = eeg.sample(n=1000) _ = pd.plotting.scatter_matrix(     X.iloc[:100, :4],     c=y[:100],     figsize=(10, 10),     marker=\"o\",     hist_kwds={\"bins\": 20},     alpha=0.8,     cmap=\"plasma\", ) <pre>/Users/eragon/.pyenv/versions/3.9.19/envs/openml/lib/python3.9/site-packages/pandas/plotting/_matplotlib/misc.py:97: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored\n  ax.scatter(\n</pre> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>Edit non-critical fields, allowed for all authorized users: description, creator, contributor, collection_date, language, citation, original_data_url, paper_url</p> In\u00a0[\u00a0]: Copied! <pre>desc = (\n    \"This data sets consists of 3 different types of irises' \"\n    \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"\n    \" stored in a 150x4 numpy.ndarray\"\n)\ndid = 128\ndata_id = edit_dataset(\n    did,\n    description=desc,\n    creator=\"R.A.Fisher\",\n    collection_date=\"1937\",\n    citation=\"The use of multiple measurements in taxonomic problems\",\n    language=\"English\",\n)\nedited_dataset = get_dataset(data_id)\nprint(f\"Edited dataset ID: {data_id}\")\n</pre> desc = (     \"This data sets consists of 3 different types of irises' \"     \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"     \" stored in a 150x4 numpy.ndarray\" ) did = 128 data_id = edit_dataset(     did,     description=desc,     creator=\"R.A.Fisher\",     collection_date=\"1937\",     citation=\"The use of multiple measurements in taxonomic problems\",     language=\"English\", ) edited_dataset = get_dataset(data_id) print(f\"Edited dataset ID: {data_id}\") <p>Editing critical fields (default_target_attribute, row_id_attribute, ignore_attribute) is allowed only for the dataset owner. Further, critical fields cannot be edited if the dataset has any tasks associated with it. To edit critical fields of a dataset (without tasks) owned by you, configure the API key: openml.config.apikey = 'FILL_IN_OPENML_API_KEY' This example here only shows a failure when trying to work on a dataset not owned by you:</p> In\u00a0[\u00a0]: Copied! <pre>try:\n    data_id = edit_dataset(1, default_target_attribute=\"shape\")\nexcept openml.exceptions.OpenMLServerException as e:\n    print(e)\n</pre> try:     data_id = edit_dataset(1, default_target_attribute=\"shape\") except openml.exceptions.OpenMLServerException as e:     print(e) In\u00a0[\u00a0]: Copied! <pre>data_id = fork_dataset(1)\nprint(data_id)\ndata_id = edit_dataset(data_id, default_target_attribute=\"shape\")\nprint(f\"Forked dataset ID: {data_id}\")\n\nopenml.config.stop_using_configuration_for_example()\n</pre> data_id = fork_dataset(1) print(data_id) data_id = edit_dataset(data_id, default_target_attribute=\"shape\") print(f\"Forked dataset ID: {data_id}\")  openml.config.stop_using_configuration_for_example()"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#datasets","title":"Datasets\u00b6","text":"<p>How to list and download datasets.</p>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#exercise-0","title":"Exercise 0\u00b6","text":"<ul> <li><p>List datasets</p> <ul> <li>Use the output_format parameter to select output type</li> <li>Default gives 'dict' (other option: 'dataframe', see below)</li> </ul> </li> </ul> <p>Note: list_datasets will return a pandas dataframe by default from 0.15. When using openml-python 0.14, <code>list_datasets</code> will warn you to use output_format='dataframe'.</p>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#exercise-1","title":"Exercise 1\u00b6","text":"<ul> <li>Find datasets with more than 10000 examples.</li> <li>Find a dataset called 'eeg_eye_state'.</li> <li>Find all datasets with more than 50 classes.</li> </ul>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#download-datasets","title":"Download datasets\u00b6","text":""},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#exercise-2","title":"Exercise 2\u00b6","text":"<ul> <li>Explore the data visually.</li> </ul>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#edit-a-created-dataset","title":"Edit a created dataset\u00b6","text":"<p>This example uses the test server, to avoid editing a dataset on the main server.</p> Warning<p>.. include:: ../../test_server_usage_warning.txt</p>"},{"location":"ecosystem/Scikit-learn/datasets_tutorial/#fork-dataset","title":"Fork dataset\u00b6","text":"<p>Used to create a copy of the dataset with you as the owner. Use this API only if you are unable to edit the critical fields (default_target_attribute, ignore_attribute, row_id_attribute) of a dataset through the edit_dataset API. After the dataset is forked, you can edit the new version of the dataset using edit_dataset.</p>"},{"location":"examples/SUMMARY/","title":"SUMMARY","text":"<ul> <li>Basic<ul> <li>introduction_tutorial.py</li> <li>simple_datasets_tutorial.py</li> <li>simple_flows_and_runs_tutorial.py</li> <li>simple_suites_tutorial.py</li> </ul> </li> <li>Extended<ul> <li>configure_logging.py</li> <li>create_upload_tutorial.py</li> <li>custom_flow_.py</li> <li>datasets_tutorial.py</li> <li>fetch_evaluations_tutorial.py</li> <li>fetch_runtimes_tutorial.py</li> <li>flow_id_tutorial.py</li> <li>flows_and_runs_tutorial.py</li> <li>plot_svm_hyperparameters_tutorial.py</li> <li>run_setup_tutorial.py</li> <li>study_tutorial.py</li> <li>suites_tutorial.py</li> <li>task_manual_iteration_tutorial.py</li> <li>tasks_tutorial.py</li> </ul> </li> <li>Paper<ul> <li>2015_neurips_feurer_example.py</li> <li>2018_ida_strang_example.py</li> <li>2018_kdd_rijn_example.py</li> <li>2018_neurips_perrone_example.py</li> </ul> </li> </ul>"},{"location":"examples/20_basic/introduction_tutorial/","title":"Introduction tutorial &amp; Setup","text":"<p>OpenML is an online collaboration platform for machine learning which allows you to:</p> <ul> <li>Find or share interesting, well-documented datasets</li> <li>Define research / modelling goals (tasks)</li> <li>Explore large amounts of machine learning algorithms, with APIs in Java, R, Python</li> <li>Log and share reproducible experiments, models, results</li> <li>Works seamlessly with scikit-learn and other libraries</li> <li>Large scale benchmarking, compare to state of the art</li> </ul> In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom sklearn import neighbors\n</pre>  import openml from sklearn import neighbors <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>When using the main server instead, make sure your apikey is configured. This can be done with the following line of code (uncomment it!). Never share your apikey with others.</p> In\u00a0[\u00a0]: Copied! <pre># openml.config.apikey = 'YOURKEY'\n</pre> # openml.config.apikey = 'YOURKEY' In\u00a0[\u00a0]: Copied! <pre># Uncomment and set your OpenML cache directory\n# import os\n# openml.config.cache_directory = os.path.expanduser('YOURDIR')\n</pre> # Uncomment and set your OpenML cache directory # import os # openml.config.cache_directory = os.path.expanduser('YOURDIR') In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(403)\ndata = openml.datasets.get_dataset(task.dataset_id)\nclf = neighbors.KNeighborsClassifier(n_neighbors=5)\nrun = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n# Publish the experiment on OpenML (optional, requires an API key).\n# For this tutorial, our configuration publishes to the test server\n# as to not crowd the main server with runs created by examples.\nmyrun = run.publish()\nprint(f\"kNN on {data.name}: {myrun.openml_url}\")\n</pre> task = openml.tasks.get_task(403) data = openml.datasets.get_dataset(task.dataset_id) clf = neighbors.KNeighborsClassifier(n_neighbors=5) run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False) # Publish the experiment on OpenML (optional, requires an API key). # For this tutorial, our configuration publishes to the test server # as to not crowd the main server with runs created by examples. myrun = run.publish() print(f\"kNN on {data.name}: {myrun.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/20_basic/introduction_tutorial/#introduction-tutorial-setup","title":"Introduction tutorial &amp; Setup\u00b6","text":"<p>An example how to set up OpenML-Python followed up by a simple example.</p>"},{"location":"examples/20_basic/introduction_tutorial/#installation","title":"Installation\u00b6","text":"<p>Installation is done via <code>pip</code>:</p> <pre>pip install openml\n</pre>"},{"location":"examples/20_basic/introduction_tutorial/#authentication","title":"Authentication\u00b6","text":"<p>The OpenML server can only be accessed by users who have signed up on the OpenML platform. If you don\u2019t have an account yet, sign up now. You will receive an API key, which will authenticate you to the server and allow you to download and upload datasets, tasks, runs and flows.</p> <ul> <li>Create an OpenML account (free) on https://www.openml.org.</li> <li>After logging in, open your account page (avatar on the top right)</li> <li>Open 'Account Settings', then 'API authentication' to find your API key.</li> </ul> <p>There are two ways to permanently authenticate:</p> <ul> <li>Use the <code>openml</code> CLI tool with <code>openml configure apikey MYKEY</code>, replacing MYKEY with your API key.</li> <li>Create a plain text file ~/.openml/config with the line 'apikey=MYKEY', replacing MYKEY with your API key. The config file must be in the directory ~/.openml/config and exist prior to importing the openml module.</li> </ul> <p>Alternatively, by running the code below and replacing 'YOURKEY' with your API key, you authenticate for the duration of the python process.</p>"},{"location":"examples/20_basic/introduction_tutorial/#caching","title":"Caching\u00b6","text":"<p>When downloading datasets, tasks, runs and flows, they will be cached to retrieve them without calling the server later. As with the API key, the cache directory can be either specified through the config file or through the API:</p> <ul> <li>Add the  line cachedir = 'MYDIR' to the config file, replacing 'MYDIR' with the path to the cache directory. By default, OpenML will use ~/.openml/cache as the cache directory.</li> <li>Run the code below, replacing 'YOURDIR' with the path to the cache directory.</li> </ul>"},{"location":"examples/20_basic/introduction_tutorial/#simple-example","title":"Simple Example\u00b6","text":"<p>Download the OpenML task for the eeg-eye-state.</p>"},{"location":"examples/20_basic/simple_datasets_tutorial/","title":"Datasets","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre>  import openml In\u00a0[\u00a0]: Copied! <pre>datasets_df = openml.datasets.list_datasets(output_format=\"dataframe\")\nprint(datasets_df.head(n=10))\n</pre> datasets_df = openml.datasets.list_datasets(output_format=\"dataframe\") print(datasets_df.head(n=10)) In\u00a0[\u00a0]: Copied! <pre># Iris dataset https://www.openml.org/d/61\ndataset = openml.datasets.get_dataset(61)\n\n# Print a summary\nprint(\n    f\"This is dataset '{dataset.name}', the target feature is \"\n    f\"'{dataset.default_target_attribute}'\"\n)\nprint(f\"URL: {dataset.url}\")\nprint(dataset.description[:500])\n</pre> # Iris dataset https://www.openml.org/d/61 dataset = openml.datasets.get_dataset(61)  # Print a summary print(     f\"This is dataset '{dataset.name}', the target feature is \"     f\"'{dataset.default_target_attribute}'\" ) print(f\"URL: {dataset.url}\") print(dataset.description[:500]) In\u00a0[\u00a0]: Copied! <pre>X, y, categorical_indicator, attribute_names = dataset.get_data(\n    dataset_format=\"dataframe\", target=dataset.default_target_attribute\n)\n</pre> X, y, categorical_indicator, attribute_names = dataset.get_data(     dataset_format=\"dataframe\", target=dataset.default_target_attribute ) <p>Visualize the dataset</p> In\u00a0[\u00a0]: Copied! <pre>import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nsns.set_style(\"darkgrid\")\n\n\ndef hide_current_axis(*args, **kwds):\n    plt.gca().set_visible(False)\n\n\n# We combine all the data so that we can map the different\n# examples to different colors according to the classes.\ncombined_data = pd.concat([X, y], axis=1)\niris_plot = sns.pairplot(combined_data, hue=\"class\")\niris_plot.map_upper(hide_current_axis)\nplt.show()\n\n# License: BSD 3-Clause\n</pre> import pandas as pd import seaborn as sns import matplotlib.pyplot as plt  sns.set_style(\"darkgrid\")   def hide_current_axis(*args, **kwds):     plt.gca().set_visible(False)   # We combine all the data so that we can map the different # examples to different colors according to the classes. combined_data = pd.concat([X, y], axis=1) iris_plot = sns.pairplot(combined_data, hue=\"class\") iris_plot.map_upper(hide_current_axis) plt.show()  # License: BSD 3-Clause"},{"location":"examples/20_basic/simple_datasets_tutorial/#datasets","title":"Datasets\u00b6","text":"<p>A basic tutorial on how to list, load and visualize datasets.</p> <p>In general, we recommend working with tasks, so that the results can be easily reproduced. Furthermore, the results can be compared to existing results at OpenML. However, for the purposes of this tutorial, we are going to work with the datasets directly.</p>"},{"location":"examples/20_basic/simple_datasets_tutorial/#list-datasets","title":"List datasets\u00b6","text":""},{"location":"examples/20_basic/simple_datasets_tutorial/#download-a-dataset","title":"Download a dataset\u00b6","text":""},{"location":"examples/20_basic/simple_datasets_tutorial/#load-a-dataset","title":"Load a dataset\u00b6","text":"<p>X - An array/dataframe where each row represents one example with the corresponding feature values.</p> <p>y - the classes for each example</p> <p>categorical_indicator - an array that indicates which feature is categorical</p> <p>attribute_names - the names of the features for the examples (X) and target feature (y)</p>"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/","title":"Flows and Runs","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom sklearn import ensemble, neighbors\n</pre> import openml from sklearn import ensemble, neighbors <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>dataset = openml.datasets.get_dataset(20)\nX, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nclf = neighbors.KNeighborsClassifier(n_neighbors=3)\nclf.fit(X, y)\n</pre> dataset = openml.datasets.get_dataset(20) X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) clf = neighbors.KNeighborsClassifier(n_neighbors=3) clf.fit(X, y) In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(119)\nclf = ensemble.RandomForestClassifier()\nrun = openml.runs.run_model_on_task(clf, task)\nprint(run)\n</pre> task = openml.tasks.get_task(119) clf = ensemble.RandomForestClassifier() run = openml.runs.run_model_on_task(clf, task) print(run) In\u00a0[\u00a0]: Copied! <pre>myrun = run.publish()\nprint(f\"Run was uploaded to {myrun.openml_url}\")\nprint(f\"The flow can be found at {myrun.flow.openml_url}\")\n</pre> myrun = run.publish() print(f\"Run was uploaded to {myrun.openml_url}\") print(f\"The flow can be found at {myrun.flow.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#flows-and-runs","title":"Flows and Runs\u00b6","text":"<p>A simple tutorial on how to train/run a model and how to upload the results.</p>"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#train-a-machine-learning-model","title":"Train a machine learning model\u00b6","text":"<p>NOTE: We are using dataset 20 from the test server: https://test.openml.org/d/20</p>"},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#running-a-model-on-a-task","title":"Running a model on a task\u00b6","text":""},{"location":"examples/20_basic/simple_flows_and_runs_tutorial/#publishing-the-run","title":"Publishing the run\u00b6","text":""},{"location":"examples/20_basic/simple_suites_tutorial/","title":"Benchmark suites","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre> import openml In\u00a0[\u00a0]: Copied! <pre>suite = openml.study.get_suite(99)\nprint(suite)\n</pre> suite = openml.study.get_suite(99) print(suite) <p>The benchmark suite does not download the included tasks and datasets itself, but only contains a list of which tasks constitute the study.</p> <p>Tasks can then be accessed via</p> In\u00a0[\u00a0]: Copied! <pre>tasks = suite.tasks\nprint(tasks)\n</pre> tasks = suite.tasks print(tasks) <p>and iterated over for benchmarking. For speed reasons we only iterate over the first three tasks:</p> In\u00a0[\u00a0]: Copied! <pre>for task_id in tasks[:3]:\n    task = openml.tasks.get_task(task_id)\n    print(task)\n</pre> for task_id in tasks[:3]:     task = openml.tasks.get_task(task_id)     print(task)"},{"location":"examples/20_basic/simple_suites_tutorial/#benchmark-suites","title":"Benchmark suites\u00b6","text":"<p>This is a brief showcase of OpenML benchmark suites, which were introduced by Bischl et al. (2019). Benchmark suites standardize the datasets and splits to be used in an experiment or paper. They are fully integrated into OpenML and simplify both the sharing of the setup and the results.</p>"},{"location":"examples/20_basic/simple_suites_tutorial/#openml-cc18","title":"OpenML-CC18\u00b6","text":"<p>As an example we have a look at the OpenML-CC18, which is a suite of 72 classification datasets from OpenML which were carefully selected to be usable by many algorithms and also represent datasets commonly used in machine learning research. These are all datasets from mid-2018 that satisfy a large set of clear requirements for thorough yet practical benchmarking:</p> <ol> <li>the number of observations are between 500 and 100,000 to focus on medium-sized datasets,</li> <li>the number of features does not exceed 5,000 features to keep the runtime of the algorithms low</li> <li>the target attribute has at least two classes with no class having less than 20 observations</li> <li>the ratio of the minority class and the majority class is above 0.05 (to eliminate highly imbalanced datasets which require special treatment for both algorithms and evaluation measures).</li> </ol> <p>A full description can be found in the OpenML benchmarking docs.</p> <p>In this example we'll focus on how to use benchmark suites in practice.</p>"},{"location":"examples/20_basic/simple_suites_tutorial/#downloading-benchmark-suites","title":"Downloading benchmark suites\u00b6","text":""},{"location":"examples/20_basic/simple_suites_tutorial/#further-examples","title":"Further examples\u00b6","text":"<ul> <li>Suites Tutorial</li> <li>Study Tutoral</li> <li>Paper example: Strang et al.</li> </ul> <p>License: BSD 3-Clause</p>"},{"location":"examples/30_extended/configure_logging/","title":"Logging","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n\nopenml.datasets.get_dataset(\"iris\")\n</pre> import openml  openml.datasets.get_dataset(\"iris\") <p>With default configuration, the above example will show no output to console. However, in your cache directory you should find a file named 'openml_python.log', which has a DEBUG message written to it. It should be either like \"[DEBUG] [10:46:19:openml.datasets.dataset] Saved dataset 61: iris to file ...\" or like \"[DEBUG] [10:49:38:openml.datasets.dataset] Data pickle file already exists and is up to date.\" , depending on whether or not you had downloaded iris before. The processed log levels can be configured programmatically:</p> In\u00a0[\u00a0]: Copied! <pre>import logging\n\nopenml.config.set_console_log_level(logging.DEBUG)\nopenml.config.set_file_log_level(logging.WARNING)\nopenml.datasets.get_dataset(\"iris\")\n</pre> import logging  openml.config.set_console_log_level(logging.DEBUG) openml.config.set_file_log_level(logging.WARNING) openml.datasets.get_dataset(\"iris\") <p>Now the log level that was previously written to file should also be shown in the console. The message is now no longer written to file as the <code>file_log</code> was set to level <code>WARNING</code>.</p> <p>It is also possible to specify the desired log levels through the configuration file. This way you will not need to set them on each script separately. Add the  line verbosity = NUMBER and/or file_verbosity = NUMBER to the config file, where 'NUMBER' should be one of:</p> <ul> <li>0: <code>logging.WARNING</code> and up.</li> <li>1: <code>logging.INFO</code> and up.</li> <li>2: <code>logging.DEBUG</code> and up (i.e. all messages).</li> </ul> <p>License: BSD 3-Clause</p>"},{"location":"examples/30_extended/configure_logging/#logging","title":"Logging\u00b6","text":"<p>This tutorial explains openml-python logging, and shows how to configure it. Openml-python uses the Python logging module to provide users with log messages. Each log message is assigned a level of importance, see the table in Python's logging tutorial here.</p> <p>By default, openml-python will print log messages of level <code>WARNING</code> and above to console. All log messages (including <code>DEBUG</code> and <code>INFO</code>) are also saved in a file, which can be found in your cache directory (see also the introduction tutorial. These file logs are automatically deleted if needed, and use at most 2MB of space.</p> <p>It is possible to configure what log levels to send to console and file. When downloading a dataset from OpenML, a <code>DEBUG</code>-level message is written:</p>"},{"location":"examples/30_extended/create_upload_tutorial/","title":"Dataset upload tutorial","text":"In\u00a0[\u00a0]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport sklearn.datasets\nfrom scipy.sparse import coo_matrix\n\nimport openml\nfrom openml.datasets.functions import create_dataset\n</pre> import numpy as np import pandas as pd import sklearn.datasets from scipy.sparse import coo_matrix  import openml from openml.datasets.functions import create_dataset <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>Below we will cover the following cases of the dataset object:</p> <ul> <li>A numpy array</li> <li>A list</li> <li>A pandas dataframe</li> <li>A sparse matrix</li> <li>A pandas sparse dataframe</li> </ul> In\u00a0[\u00a0]: Copied! <pre>diabetes = sklearn.datasets.load_diabetes()\nname = \"Diabetes(scikit-learn)\"\nX = diabetes.data\ny = diabetes.target\nattribute_names = diabetes.feature_names\ndescription = diabetes.DESCR\n</pre> diabetes = sklearn.datasets.load_diabetes() name = \"Diabetes(scikit-learn)\" X = diabetes.data y = diabetes.target attribute_names = diabetes.feature_names description = diabetes.DESCR <p>OpenML does not distinguish between the attributes and targets on the data level and stores all data in a single matrix.</p> <p>The target feature is indicated as meta-data of the dataset (and tasks on that data).</p> In\u00a0[\u00a0]: Copied! <pre>data = np.concatenate((X, y.reshape((-1, 1))), axis=1)\nattribute_names = list(attribute_names)\nattributes = [(attribute_name, \"REAL\") for attribute_name in attribute_names] + [\n    (\"class\", \"INTEGER\")\n]\ncitation = (\n    \"Bradley Efron, Trevor Hastie, Iain Johnstone and \"\n    \"Robert Tibshirani (2004) (Least Angle Regression) \"\n    \"Annals of Statistics (with discussion), 407-499\"\n)\npaper_url = \"https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf\"\n</pre> data = np.concatenate((X, y.reshape((-1, 1))), axis=1) attribute_names = list(attribute_names) attributes = [(attribute_name, \"REAL\") for attribute_name in attribute_names] + [     (\"class\", \"INTEGER\") ] citation = (     \"Bradley Efron, Trevor Hastie, Iain Johnstone and \"     \"Robert Tibshirani (2004) (Least Angle Regression) \"     \"Annals of Statistics (with discussion), 407-499\" ) paper_url = \"https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf\" In\u00a0[\u00a0]: Copied! <pre>diabetes_dataset = create_dataset(\n    # The name of the dataset (needs to be unique).\n    # Must not be longer than 128 characters and only contain\n    # a-z, A-Z, 0-9 and the following special characters: _\\-\\.(),\n    name=name,\n    # Textual description of the dataset.\n    description=description,\n    # The person who created the dataset.\n    creator=\"Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani\",\n    # People who contributed to the current version of the dataset.\n    contributor=None,\n    # The date the data was originally collected, given by the uploader.\n    collection_date=\"09-01-2012\",\n    # Language in which the data is represented.\n    # Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    language=\"English\",\n    # License under which the data is/will be distributed.\n    licence=\"BSD (from scikit-learn)\",\n    # Name of the target. Can also have multiple values (comma-separated).\n    default_target_attribute=\"class\",\n    # The attribute that represents the row-id column, if present in the\n    # dataset.\n    row_id_attribute=None,\n    # Attribute or list of attributes that should be excluded in modelling, such as\n    # identifiers and indexes. E.g. \"feat1\" or [\"feat1\",\"feat2\"]\n    ignore_attribute=None,\n    # How to cite the paper.\n    citation=citation,\n    # Attributes of the data\n    attributes=attributes,\n    data=data,\n    # A version label which is provided by the user.\n    version_label=\"test\",\n    original_data_url=\"https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html\",\n    paper_url=paper_url,\n)\n</pre> diabetes_dataset = create_dataset(     # The name of the dataset (needs to be unique).     # Must not be longer than 128 characters and only contain     # a-z, A-Z, 0-9 and the following special characters: _\\-\\.(),     name=name,     # Textual description of the dataset.     description=description,     # The person who created the dataset.     creator=\"Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani\",     # People who contributed to the current version of the dataset.     contributor=None,     # The date the data was originally collected, given by the uploader.     collection_date=\"09-01-2012\",     # Language in which the data is represented.     # Starts with 1 upper case letter, rest lower case, e.g. 'English'.     language=\"English\",     # License under which the data is/will be distributed.     licence=\"BSD (from scikit-learn)\",     # Name of the target. Can also have multiple values (comma-separated).     default_target_attribute=\"class\",     # The attribute that represents the row-id column, if present in the     # dataset.     row_id_attribute=None,     # Attribute or list of attributes that should be excluded in modelling, such as     # identifiers and indexes. E.g. \"feat1\" or [\"feat1\",\"feat2\"]     ignore_attribute=None,     # How to cite the paper.     citation=citation,     # Attributes of the data     attributes=attributes,     data=data,     # A version label which is provided by the user.     version_label=\"test\",     original_data_url=\"https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html\",     paper_url=paper_url, ) In\u00a0[\u00a0]: Copied! <pre>diabetes_dataset.publish()\nprint(f\"URL for dataset: {diabetes_dataset.openml_url}\")\n</pre>  diabetes_dataset.publish() print(f\"URL for dataset: {diabetes_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>data = [\n    [\"sunny\", 85, 85, \"FALSE\", \"no\"],\n    [\"sunny\", 80, 90, \"TRUE\", \"no\"],\n    [\"overcast\", 83, 86, \"FALSE\", \"yes\"],\n    [\"rainy\", 70, 96, \"FALSE\", \"yes\"],\n    [\"rainy\", 68, 80, \"FALSE\", \"yes\"],\n    [\"rainy\", 65, 70, \"TRUE\", \"no\"],\n    [\"overcast\", 64, 65, \"TRUE\", \"yes\"],\n    [\"sunny\", 72, 95, \"FALSE\", \"no\"],\n    [\"sunny\", 69, 70, \"FALSE\", \"yes\"],\n    [\"rainy\", 75, 80, \"FALSE\", \"yes\"],\n    [\"sunny\", 75, 70, \"TRUE\", \"yes\"],\n    [\"overcast\", 72, 90, \"TRUE\", \"yes\"],\n    [\"overcast\", 81, 75, \"FALSE\", \"yes\"],\n    [\"rainy\", 71, 91, \"TRUE\", \"no\"],\n]\n\nattribute_names = [\n    (\"outlook\", [\"sunny\", \"overcast\", \"rainy\"]),\n    (\"temperature\", \"REAL\"),\n    (\"humidity\", \"REAL\"),\n    (\"windy\", [\"TRUE\", \"FALSE\"]),\n    (\"play\", [\"yes\", \"no\"]),\n]\n\ndescription = (\n    \"The weather problem is a tiny dataset that we will use repeatedly\"\n    \" to illustrate machine learning methods. Entirely fictitious, it \"\n    \"supposedly concerns the conditions that are suitable for playing \"\n    \"some unspecified game. In general, instances in a dataset are \"\n    \"characterized by the values of features, or attributes, that measure \"\n    \"different aspects of the instance. In this case there are four \"\n    \"attributes: outlook, temperature, humidity, and windy. \"\n    \"The outcome is whether to play or not.\"\n)\n\ncitation = (\n    \"I. H. Witten, E. Frank, M. A. Hall, and ITPro,\"\n    \"Data mining practical machine learning tools and techniques, \"\n    \"third edition. Burlington, Mass.: Morgan Kaufmann Publishers, 2011\"\n)\n\nweather_dataset = create_dataset(\n    name=\"Weather\",\n    description=description,\n    creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",\n    contributor=None,\n    collection_date=\"01-01-2011\",\n    language=\"English\",\n    licence=None,\n    default_target_attribute=\"play\",\n    row_id_attribute=None,\n    ignore_attribute=None,\n    citation=citation,\n    attributes=attribute_names,\n    data=data,\n    version_label=\"example\",\n)\n</pre> data = [     [\"sunny\", 85, 85, \"FALSE\", \"no\"],     [\"sunny\", 80, 90, \"TRUE\", \"no\"],     [\"overcast\", 83, 86, \"FALSE\", \"yes\"],     [\"rainy\", 70, 96, \"FALSE\", \"yes\"],     [\"rainy\", 68, 80, \"FALSE\", \"yes\"],     [\"rainy\", 65, 70, \"TRUE\", \"no\"],     [\"overcast\", 64, 65, \"TRUE\", \"yes\"],     [\"sunny\", 72, 95, \"FALSE\", \"no\"],     [\"sunny\", 69, 70, \"FALSE\", \"yes\"],     [\"rainy\", 75, 80, \"FALSE\", \"yes\"],     [\"sunny\", 75, 70, \"TRUE\", \"yes\"],     [\"overcast\", 72, 90, \"TRUE\", \"yes\"],     [\"overcast\", 81, 75, \"FALSE\", \"yes\"],     [\"rainy\", 71, 91, \"TRUE\", \"no\"], ]  attribute_names = [     (\"outlook\", [\"sunny\", \"overcast\", \"rainy\"]),     (\"temperature\", \"REAL\"),     (\"humidity\", \"REAL\"),     (\"windy\", [\"TRUE\", \"FALSE\"]),     (\"play\", [\"yes\", \"no\"]), ]  description = (     \"The weather problem is a tiny dataset that we will use repeatedly\"     \" to illustrate machine learning methods. Entirely fictitious, it \"     \"supposedly concerns the conditions that are suitable for playing \"     \"some unspecified game. In general, instances in a dataset are \"     \"characterized by the values of features, or attributes, that measure \"     \"different aspects of the instance. In this case there are four \"     \"attributes: outlook, temperature, humidity, and windy. \"     \"The outcome is whether to play or not.\" )  citation = (     \"I. H. Witten, E. Frank, M. A. Hall, and ITPro,\"     \"Data mining practical machine learning tools and techniques, \"     \"third edition. Burlington, Mass.: Morgan Kaufmann Publishers, 2011\" )  weather_dataset = create_dataset(     name=\"Weather\",     description=description,     creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",     contributor=None,     collection_date=\"01-01-2011\",     language=\"English\",     licence=None,     default_target_attribute=\"play\",     row_id_attribute=None,     ignore_attribute=None,     citation=citation,     attributes=attribute_names,     data=data,     version_label=\"example\", ) In\u00a0[\u00a0]: Copied! <pre>weather_dataset.publish()\nprint(f\"URL for dataset: {weather_dataset.openml_url}\")\n</pre> weather_dataset.publish() print(f\"URL for dataset: {weather_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>df = pd.DataFrame(data, columns=[col_name for col_name, _ in attribute_names])\n\n# enforce the categorical column to have a categorical dtype\ndf[\"outlook\"] = df[\"outlook\"].astype(\"category\")\ndf[\"windy\"] = df[\"windy\"].astype(\"bool\")\ndf[\"play\"] = df[\"play\"].astype(\"category\")\nprint(df.info())\n</pre> df = pd.DataFrame(data, columns=[col_name for col_name, _ in attribute_names])  # enforce the categorical column to have a categorical dtype df[\"outlook\"] = df[\"outlook\"].astype(\"category\") df[\"windy\"] = df[\"windy\"].astype(\"bool\") df[\"play\"] = df[\"play\"].astype(\"category\") print(df.info()) <p>We enforce the column 'outlook' and 'play' to be a categorical dtype while the column 'windy' is kept as a boolean column. 'temperature' and 'humidity' are kept as numeric columns. Then, we can call :func:<code>openml.datasets.create_dataset</code> by passing the dataframe and fixing the parameter <code>attributes</code> to <code>'auto'</code>.</p> In\u00a0[\u00a0]: Copied! <pre>weather_dataset = create_dataset(\n    name=\"Weather\",\n    description=description,\n    creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",\n    contributor=None,\n    collection_date=\"01-01-2011\",\n    language=\"English\",\n    licence=None,\n    default_target_attribute=\"play\",\n    row_id_attribute=None,\n    ignore_attribute=None,\n    citation=citation,\n    attributes=\"auto\",\n    data=df,\n    version_label=\"example\",\n)\n</pre> weather_dataset = create_dataset(     name=\"Weather\",     description=description,     creator=\"I. H. Witten, E. Frank, M. A. Hall, and ITPro\",     contributor=None,     collection_date=\"01-01-2011\",     language=\"English\",     licence=None,     default_target_attribute=\"play\",     row_id_attribute=None,     ignore_attribute=None,     citation=citation,     attributes=\"auto\",     data=df,     version_label=\"example\", ) In\u00a0[\u00a0]: Copied! <pre>weather_dataset.publish()\nprint(f\"URL for dataset: {weather_dataset.openml_url}\")\n</pre> weather_dataset.publish() print(f\"URL for dataset: {weather_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>sparse_data = coo_matrix(\n    ([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))\n)\n\ncolumn_names = [\n    (\"input1\", \"REAL\"),\n    (\"input2\", \"REAL\"),\n    (\"y\", \"REAL\"),\n]\n\nxor_dataset = create_dataset(\n    name=\"XOR\",\n    description=\"Dataset representing the XOR operation\",\n    creator=None,\n    contributor=None,\n    collection_date=None,\n    language=\"English\",\n    licence=None,\n    default_target_attribute=\"y\",\n    row_id_attribute=None,\n    ignore_attribute=None,\n    citation=None,\n    attributes=column_names,\n    data=sparse_data,\n    version_label=\"example\",\n)\n</pre> sparse_data = coo_matrix(     ([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1])) )  column_names = [     (\"input1\", \"REAL\"),     (\"input2\", \"REAL\"),     (\"y\", \"REAL\"), ]  xor_dataset = create_dataset(     name=\"XOR\",     description=\"Dataset representing the XOR operation\",     creator=None,     contributor=None,     collection_date=None,     language=\"English\",     licence=None,     default_target_attribute=\"y\",     row_id_attribute=None,     ignore_attribute=None,     citation=None,     attributes=column_names,     data=sparse_data,     version_label=\"example\", ) In\u00a0[\u00a0]: Copied! <pre>xor_dataset.publish()\nprint(f\"URL for dataset: {xor_dataset.openml_url}\")\n</pre> xor_dataset.publish() print(f\"URL for dataset: {xor_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>xor_dataset.publish()\nprint(f\"URL for dataset: {xor_dataset.openml_url}\")\n</pre>  xor_dataset.publish() print(f\"URL for dataset: {xor_dataset.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-upload-tutorial","title":"Dataset upload tutorial\u00b6","text":"<p>A tutorial on how to create and upload a dataset to OpenML.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-numpy-array","title":"Dataset is a numpy array\u00b6","text":"<p>A numpy array can contain lists in the case of dense data or it can contain OrderedDicts in the case of sparse data.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#prepare-dataset","title":"Prepare dataset\u00b6","text":"<p>Load an example dataset from scikit-learn which we will upload to OpenML.org via the API.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#create-the-dataset-object","title":"Create the dataset object\u00b6","text":"<p>The definition of all fields can be found in the XSD files describing the expected format:</p> <p>https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.data.upload.xsd</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-list","title":"Dataset is a list\u00b6","text":"<p>A list can contain lists in the case of dense data or it can contain OrderedDicts in the case of sparse data.</p> <p>Weather dataset: https://storm.cis.fordham.edu/~gweiss/data-mining/datasets.html</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-pandas-dataframe","title":"Dataset is a pandas DataFrame\u00b6","text":"<p>It might happen that your dataset is made of heterogeneous data which can usually be stored as a Pandas DataFrame. DataFrames offer the advantage of storing the type of data for each column as well as the attribute names. Therefore, when providing a Pandas DataFrame, OpenML can infer this information without needing to explicitly provide it when calling the function :func:<code>openml.datasets.create_dataset</code>. In this regard, you only need to pass <code>'auto'</code> to the <code>attributes</code> parameter.</p>"},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-sparse-matrix","title":"Dataset is a sparse matrix\u00b6","text":""},{"location":"examples/30_extended/create_upload_tutorial/#dataset-is-a-pandas-dataframe-with-sparse-columns","title":"Dataset is a pandas dataframe with sparse columns\u00b6","text":"<p>sparse_data = coo_matrix( ([1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1])) ) column_names = [\"input1\", \"input2\", \"y\"] df = pd.DataFrame.sparse.from_spmatrix(sparse_data, columns=column_names) print(df.info())</p> <p>xor_dataset = create_dataset( name=\"XOR\", description=\"Dataset representing the XOR operation\", creator=None, contributor=None, collection_date=None, language=\"English\", licence=None, default_target_attribute=\"y\", row_id_attribute=None, ignore_attribute=None, citation=None, attributes=\"auto\", data=df, version_label=\"example\", )</p>"},{"location":"examples/30_extended/custom_flow_/","title":"Creating and Using a Custom Flow","text":"In\u00a0[\u00a0]: Copied! <pre>from collections import OrderedDict\nimport numpy as np\n\nimport openml\nfrom openml import OpenMLClassificationTask\nfrom openml.runs.functions import format_prediction\n</pre> from collections import OrderedDict import numpy as np  import openml from openml import OpenMLClassificationTask from openml.runs.functions import format_prediction <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>general = dict(\n    name=\"automlbenchmark_autosklearn\",\n    description=(\n        \"Auto-sklearn as set up by the AutoML Benchmark\"\n        \"Source: https://github.com/openml/automlbenchmark/releases/tag/v0.9\"\n    ),\n    external_version=\"amlb==0.9\",\n    language=\"English\",\n    tags=[\"amlb\", \"benchmark\", \"study_218\"],\n    dependencies=\"amlb==0.9\",\n)\n</pre> general = dict(     name=\"automlbenchmark_autosklearn\",     description=(         \"Auto-sklearn as set up by the AutoML Benchmark\"         \"Source: https://github.com/openml/automlbenchmark/releases/tag/v0.9\"     ),     external_version=\"amlb==0.9\",     language=\"English\",     tags=[\"amlb\", \"benchmark\", \"study_218\"],     dependencies=\"amlb==0.9\", ) <p>Next we define the flow hyperparameters. We define their name and default value in <code>parameters</code>, and provide meta-data for each hyperparameter through <code>parameters_meta_info</code>. Note that even though the argument name is <code>parameters</code> they describe the hyperparameters. The use of ordered dicts is required.</p> In\u00a0[\u00a0]: Copied! <pre>flow_hyperparameters = dict(\n    parameters=OrderedDict(time=\"240\", memory=\"32\", cores=\"8\"),\n    parameters_meta_info=OrderedDict(\n        cores=OrderedDict(description=\"number of available cores\", data_type=\"int\"),\n        memory=OrderedDict(description=\"memory in gigabytes\", data_type=\"int\"),\n        time=OrderedDict(description=\"time in minutes\", data_type=\"int\"),\n    ),\n)\n</pre> flow_hyperparameters = dict(     parameters=OrderedDict(time=\"240\", memory=\"32\", cores=\"8\"),     parameters_meta_info=OrderedDict(         cores=OrderedDict(description=\"number of available cores\", data_type=\"int\"),         memory=OrderedDict(description=\"memory in gigabytes\", data_type=\"int\"),         time=OrderedDict(description=\"time in minutes\", data_type=\"int\"),     ), ) <p>It is possible to build a flow which uses other flows. For example, the Random Forest Classifier is a flow, but you could also construct a flow which uses a Random Forest Classifier in a ML pipeline. When constructing the pipeline flow, you can use the Random Forest Classifier flow as a subflow. It allows for all hyperparameters of the Random Classifier Flow to also be specified in your pipeline flow.</p> <p>Note: you can currently only specific one subflow as part of the components.</p> <p>In this example, the auto-sklearn flow is a subflow: the auto-sklearn flow is entirely executed as part of this flow. This allows people to specify auto-sklearn hyperparameters used in this flow. In general, using a subflow is not required.</p> <p>Note: flow 9313 is not actually the right flow on the test server, but that does not matter for this demonstration.</p> In\u00a0[\u00a0]: Copied! <pre>autosklearn_flow = openml.flows.get_flow(9313)  # auto-sklearn 0.5.1\nsubflow = dict(\n    components=OrderedDict(automl_tool=autosklearn_flow),\n    # If you do not want to reference a subflow, you can use the following:\n    # components=OrderedDict(),\n)\n</pre> autosklearn_flow = openml.flows.get_flow(9313)  # auto-sklearn 0.5.1 subflow = dict(     components=OrderedDict(automl_tool=autosklearn_flow),     # If you do not want to reference a subflow, you can use the following:     # components=OrderedDict(), ) <p>With all parameters of the flow defined, we can now initialize the OpenMLFlow and publish. Because we provided all the details already, we do not need to provide a <code>model</code> to the flow.</p> <p>In our case, we don't even have a model. It is possible to have a model but still require to follow these steps when the model (python object) does not have an extensions from which to automatically extract the hyperparameters. So whether you have a model with no extension or no model at all, explicitly set the model of the flow to <code>None</code>.</p> In\u00a0[\u00a0]: Copied! <pre>autosklearn_amlb_flow = openml.flows.OpenMLFlow(\n    **general,\n    **flow_hyperparameters,\n    **subflow,\n    model=None,\n)\nautosklearn_amlb_flow.publish()\nprint(f\"autosklearn flow created: {autosklearn_amlb_flow.flow_id}\")\n</pre> autosklearn_amlb_flow = openml.flows.OpenMLFlow(     **general,     **flow_hyperparameters,     **subflow,     model=None, ) autosklearn_amlb_flow.publish() print(f\"autosklearn flow created: {autosklearn_amlb_flow.flow_id}\") In\u00a0[\u00a0]: Copied! <pre>flow_id = autosklearn_amlb_flow.flow_id\n\nparameters = [\n    OrderedDict([(\"oml:name\", \"cores\"), (\"oml:value\", 4), (\"oml:component\", flow_id)]),\n    OrderedDict([(\"oml:name\", \"memory\"), (\"oml:value\", 16), (\"oml:component\", flow_id)]),\n    OrderedDict([(\"oml:name\", \"time\"), (\"oml:value\", 120), (\"oml:component\", flow_id)]),\n]\n\ntask_id = 1200  # Iris Task\ntask = openml.tasks.get_task(task_id)\ndataset_id = task.get_dataset().dataset_id\n</pre> flow_id = autosklearn_amlb_flow.flow_id  parameters = [     OrderedDict([(\"oml:name\", \"cores\"), (\"oml:value\", 4), (\"oml:component\", flow_id)]),     OrderedDict([(\"oml:name\", \"memory\"), (\"oml:value\", 16), (\"oml:component\", flow_id)]),     OrderedDict([(\"oml:name\", \"time\"), (\"oml:value\", 120), (\"oml:component\", flow_id)]), ]  task_id = 1200  # Iris Task task = openml.tasks.get_task(task_id) dataset_id = task.get_dataset().dataset_id <p>The last bit of information for the run we need are the predicted values. The exact format of the predictions will depend on the task.</p> <p>The predictions should always be a list of lists, each list should contain:</p> <ul> <li>the repeat number: for repeated evaluation strategies. (e.g. repeated cross-validation)</li> <li>the fold number: for cross-validation. (what should this be for holdout?)</li> <li>0: this field is for backward compatibility.</li> <li>index: the row (of the original dataset) for which the prediction was made.</li> <li>p_1, ..., p_c: for each class the predicted probability of the sample belonging to that class. (no elements for regression tasks) Make sure the order of these elements follows the order of <code>task.class_labels</code>.</li> <li>the predicted class/value for the sample</li> <li>the true class/value for the sample</li> </ul> <p>When using openml-python extensions (such as through <code>run_model_on_task</code>), all of this formatting is automatic. Unfortunately we can not automate this procedure for custom flows, which means a little additional effort is required.</p> <p>Here we generated some random predictions in place. You can ignore this code, or use it to better understand the formatting of the predictions.</p> <p>Find the repeats/folds for this task:</p> In\u00a0[\u00a0]: Copied! <pre>n_repeats, n_folds, _ = task.get_split_dimensions()\nall_test_indices = [\n    (repeat, fold, index)\n    for repeat in range(n_repeats)\n    for fold in range(n_folds)\n    for index in task.get_train_test_split_indices(fold, repeat)[1]\n]\n\n# random class probabilities (Iris has 150 samples and 3 classes):\nr = np.random.rand(150 * n_repeats, 3)\n# scale the random values so that the probabilities of each sample sum to 1:\ny_proba = r / r.sum(axis=1).reshape(-1, 1)\ny_pred = y_proba.argmax(axis=1)\n\nclass_map = dict(zip(range(3), task.class_labels))\n_, y_true = task.get_X_and_y()\ny_true = [class_map[y] for y in y_true]\n\n# We format the predictions with the utility function `format_prediction`.\n# It will organize the relevant data in the expected format/order.\npredictions = []\nfor where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba):\n    repeat, fold, index = where\n\n    prediction = format_prediction(\n        task=task,\n        repeat=repeat,\n        fold=fold,\n        index=index,\n        prediction=class_map[yp],\n        truth=y,\n        proba={c: pb for (c, pb) in zip(task.class_labels, proba)},\n    )\n    predictions.append(prediction)\n</pre> n_repeats, n_folds, _ = task.get_split_dimensions() all_test_indices = [     (repeat, fold, index)     for repeat in range(n_repeats)     for fold in range(n_folds)     for index in task.get_train_test_split_indices(fold, repeat)[1] ]  # random class probabilities (Iris has 150 samples and 3 classes): r = np.random.rand(150 * n_repeats, 3) # scale the random values so that the probabilities of each sample sum to 1: y_proba = r / r.sum(axis=1).reshape(-1, 1) y_pred = y_proba.argmax(axis=1)  class_map = dict(zip(range(3), task.class_labels)) _, y_true = task.get_X_and_y() y_true = [class_map[y] for y in y_true]  # We format the predictions with the utility function `format_prediction`. # It will organize the relevant data in the expected format/order. predictions = [] for where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba):     repeat, fold, index = where      prediction = format_prediction(         task=task,         repeat=repeat,         fold=fold,         index=index,         prediction=class_map[yp],         truth=y,         proba={c: pb for (c, pb) in zip(task.class_labels, proba)},     )     predictions.append(prediction) <p>Finally we can create the OpenMLRun object and upload. We use the argument setup_string because the used flow was a script.</p> In\u00a0[\u00a0]: Copied! <pre>benchmark_command = f\"python3 runbenchmark.py auto-sklearn medium -m aws -t 119\"\nmy_run = openml.runs.OpenMLRun(\n    task_id=task_id,\n    flow_id=flow_id,\n    dataset_id=dataset_id,\n    parameter_settings=parameters,\n    setup_string=benchmark_command,\n    data_content=predictions,\n    tags=[\"study_218\"],\n    description_text=\"Run generated by the Custom Flow tutorial.\",\n)\nmy_run.publish()\nprint(\"run created:\", my_run.run_id)\n</pre> benchmark_command = f\"python3 runbenchmark.py auto-sklearn medium -m aws -t 119\" my_run = openml.runs.OpenMLRun(     task_id=task_id,     flow_id=flow_id,     dataset_id=dataset_id,     parameter_settings=parameters,     setup_string=benchmark_command,     data_content=predictions,     tags=[\"study_218\"],     description_text=\"Run generated by the Custom Flow tutorial.\", ) my_run.publish() print(\"run created:\", my_run.run_id) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/custom_flow_/#creating-and-using-a-custom-flow","title":"Creating and Using a Custom Flow\u00b6","text":"<p>The most convenient way to create a flow for your machine learning workflow is to generate it automatically as described in the \"Obtaining Flow IDs\" tutorial. However, there are scenarios where this is not possible, such as when the flow uses a framework without an extension or when the flow is described by a script.</p> <p>In those cases you can still create a custom flow by following the steps of this tutorial. As an example we will use the flows generated for the AutoML Benchmark, and also show how to link runs to the custom flow.</p>"},{"location":"examples/30_extended/custom_flow_/#1-defining-the-flow","title":"1. Defining the flow\u00b6","text":"<p>The first step is to define all the hyperparameters of your flow. The API pages feature a descriptions of each variable of the :class:<code>openml.flows.OpenMLFlow</code>. Note that <code>external version</code> and <code>name</code> together uniquely identify a flow.</p> <p>The AutoML Benchmark runs AutoML systems across a range of tasks. OpenML stores Flows for each AutoML system. However, the AutoML benchmark adds preprocessing to the flow, so should be described in a new flow.</p> <p>We will break down the flow arguments into several groups, for the tutorial. First we will define the name and version information. Make sure to leave enough information so others can determine exactly which version of the package/script is used. Use tags so users can find your flow easily.</p>"},{"location":"examples/30_extended/custom_flow_/#2-using-the-flow","title":"2. Using the flow\u00b6","text":"<p>This Section will show how to upload run data for your custom flow. Take care to change the values of parameters as well as the task id, to reflect the actual run. Task and parameter values in the example are fictional.</p>"},{"location":"examples/30_extended/datasets_tutorial/","title":"Datasets","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nimport pandas as pd\nfrom openml.datasets import edit_dataset, fork_dataset, get_dataset\n</pre> import openml import pandas as pd from openml.datasets import edit_dataset, fork_dataset, get_dataset In\u00a0[\u00a0]: Copied! <pre>datalist = openml.datasets.list_datasets()\ndatalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]\n\nprint(f\"First 10 of {len(datalist)} datasets...\")\ndatalist.head(n=10)\n\n# The same can be done with lesser lines of code\nopenml_df = openml.datasets.list_datasets()\nopenml_df.head(n=10)\n</pre> datalist = openml.datasets.list_datasets() datalist = datalist[[\"did\", \"name\", \"NumberOfInstances\", \"NumberOfFeatures\", \"NumberOfClasses\"]]  print(f\"First 10 of {len(datalist)} datasets...\") datalist.head(n=10)  # The same can be done with lesser lines of code openml_df = openml.datasets.list_datasets() openml_df.head(n=10) In\u00a0[\u00a0]: Copied! <pre>datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20)\n</pre> datalist[datalist.NumberOfInstances &gt; 10000].sort_values([\"NumberOfInstances\"]).head(n=20) In\u00a0[\u00a0]: Copied! <pre>datalist.query('name == \"eeg-eye-state\"')\n</pre> datalist.query('name == \"eeg-eye-state\"') In\u00a0[\u00a0]: Copied! <pre>datalist.query(\"NumberOfClasses &gt; 50\")\n</pre> datalist.query(\"NumberOfClasses &gt; 50\") In\u00a0[\u00a0]: Copied! <pre># This is done based on the dataset ID.\ndataset = openml.datasets.get_dataset(1471)\n\n# Print a summary\nprint(\n    f\"This is dataset '{dataset.name}', the target feature is \"\n    f\"'{dataset.default_target_attribute}'\"\n)\nprint(f\"URL: {dataset.url}\")\nprint(dataset.description[:500])\n</pre> # This is done based on the dataset ID. dataset = openml.datasets.get_dataset(1471)  # Print a summary print(     f\"This is dataset '{dataset.name}', the target feature is \"     f\"'{dataset.default_target_attribute}'\" ) print(f\"URL: {dataset.url}\") print(dataset.description[:500]) <p>Get the actual data.</p> <p>openml-python returns data as pandas dataframes (stored in the <code>eeg</code> variable below), and also some additional metadata that we don't care about right now.</p> In\u00a0[\u00a0]: Copied! <pre>eeg, *_ = dataset.get_data()\n</pre> eeg, *_ = dataset.get_data() <p>You can optionally choose to have openml separate out a column from the dataset. In particular, many datasets for supervised problems have a set <code>default_target_attribute</code> which may help identify the target variable.</p> In\u00a0[\u00a0]: Copied! <pre>X, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nprint(X.head())\nprint(X.info())\n</pre> X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) print(X.head()) print(X.info()) <p>Sometimes you only need access to a dataset's metadata. In those cases, you can download the dataset without downloading the data file. The dataset object can be used as normal. Whenever you use any functionality that requires the data, such as <code>get_data</code>, the data will be downloaded. Starting from 0.15, not downloading data will be the default behavior instead. The data will be downloading automatically when you try to access it through openml objects, e.g., using <code>dataset.features</code>.</p> In\u00a0[\u00a0]: Copied! <pre>dataset = openml.datasets.get_dataset(1471)\n</pre> dataset = openml.datasets.get_dataset(1471) In\u00a0[\u00a0]: Copied! <pre>eegs = eeg.sample(n=1000)\n_ = pd.plotting.scatter_matrix(\n    X.iloc[:100, :4],\n    c=y[:100],\n    figsize=(10, 10),\n    marker=\"o\",\n    hist_kwds={\"bins\": 20},\n    alpha=0.8,\n    cmap=\"plasma\",\n)\n</pre> eegs = eeg.sample(n=1000) _ = pd.plotting.scatter_matrix(     X.iloc[:100, :4],     c=y[:100],     figsize=(10, 10),     marker=\"o\",     hist_kwds={\"bins\": 20},     alpha=0.8,     cmap=\"plasma\", ) In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <p>Edit non-critical fields, allowed for all authorized users: description, creator, contributor, collection_date, language, citation, original_data_url, paper_url</p> In\u00a0[\u00a0]: Copied! <pre>desc = (\n    \"This data sets consists of 3 different types of irises' \"\n    \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"\n    \" stored in a 150x4 numpy.ndarray\"\n)\ndid = 128\ndata_id = edit_dataset(\n    did,\n    description=desc,\n    creator=\"R.A.Fisher\",\n    collection_date=\"1937\",\n    citation=\"The use of multiple measurements in taxonomic problems\",\n    language=\"English\",\n)\nedited_dataset = get_dataset(data_id)\nprint(f\"Edited dataset ID: {data_id}\")\n</pre> desc = (     \"This data sets consists of 3 different types of irises' \"     \"(Setosa, Versicolour, and Virginica) petal and sepal length,\"     \" stored in a 150x4 numpy.ndarray\" ) did = 128 data_id = edit_dataset(     did,     description=desc,     creator=\"R.A.Fisher\",     collection_date=\"1937\",     citation=\"The use of multiple measurements in taxonomic problems\",     language=\"English\", ) edited_dataset = get_dataset(data_id) print(f\"Edited dataset ID: {data_id}\") <p>Editing critical fields (default_target_attribute, row_id_attribute, ignore_attribute) is allowed only for the dataset owner. Further, critical fields cannot be edited if the dataset has any tasks associated with it. To edit critical fields of a dataset (without tasks) owned by you, configure the API key: openml.config.apikey = 'FILL_IN_OPENML_API_KEY' This example here only shows a failure when trying to work on a dataset not owned by you:</p> In\u00a0[\u00a0]: Copied! <pre>try:\n    data_id = edit_dataset(1, default_target_attribute=\"shape\")\nexcept openml.exceptions.OpenMLServerException as e:\n    print(e)\n</pre> try:     data_id = edit_dataset(1, default_target_attribute=\"shape\") except openml.exceptions.OpenMLServerException as e:     print(e) In\u00a0[\u00a0]: Copied! <pre>data_id = fork_dataset(1)\nprint(data_id)\ndata_id = edit_dataset(data_id, default_target_attribute=\"shape\")\nprint(f\"Forked dataset ID: {data_id}\")\n</pre> data_id = fork_dataset(1) print(data_id) data_id = edit_dataset(data_id, default_target_attribute=\"shape\") print(f\"Forked dataset ID: {data_id}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clauses\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clauses"},{"location":"examples/30_extended/datasets_tutorial/#datasets","title":"Datasets\u00b6","text":"<p>How to list and download datasets.</p>"},{"location":"examples/30_extended/datasets_tutorial/#exercise-0","title":"Exercise 0\u00b6","text":"<ul> <li>List datasets</li> </ul>"},{"location":"examples/30_extended/datasets_tutorial/#exercise-1","title":"Exercise 1\u00b6","text":"<ul> <li>Find datasets with more than 10000 examples.</li> <li>Find a dataset called 'eeg_eye_state'.</li> <li>Find all datasets with more than 50 classes.</li> </ul>"},{"location":"examples/30_extended/datasets_tutorial/#download-datasets","title":"Download datasets\u00b6","text":""},{"location":"examples/30_extended/datasets_tutorial/#exercise-2","title":"Exercise 2\u00b6","text":"<ul> <li>Explore the data visually.</li> </ul>"},{"location":"examples/30_extended/datasets_tutorial/#edit-a-created-dataset","title":"Edit a created dataset\u00b6","text":"<p>This example uses the test server, to avoid editing a dataset on the main server.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p>"},{"location":"examples/30_extended/datasets_tutorial/#fork-dataset","title":"Fork dataset\u00b6","text":"<p>Used to create a copy of the dataset with you as the owner. Use this API only if you are unable to edit the critical fields (default_target_attribute, ignore_attribute, row_id_attribute) of a dataset through the edit_dataset API. After the dataset is forked, you can edit the new version of the dataset using edit_dataset.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/","title":"Fetching Evaluations","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre> import openml In\u00a0[\u00a0]: Copied! <pre>openml.evaluations.list_evaluations(\n    function=\"predictive_accuracy\", size=10\n)\n\n# Using other evaluation metrics, 'precision' in this case\nevals = openml.evaluations.list_evaluations(\n    function=\"precision\", size=10\n)\n\n# Querying the returned results for precision above 0.98\nprint(evals[evals.value &gt; 0.98])\n</pre> openml.evaluations.list_evaluations(     function=\"predictive_accuracy\", size=10 )  # Using other evaluation metrics, 'precision' in this case evals = openml.evaluations.list_evaluations(     function=\"precision\", size=10 )  # Querying the returned results for precision above 0.98 print(evals[evals.value &gt; 0.98]) In\u00a0[\u00a0]: Copied! <pre>task_id = 167140  # https://www.openml.org/t/167140\ntask = openml.tasks.get_task(task_id)\nprint(task)\n</pre> task_id = 167140  # https://www.openml.org/t/167140 task = openml.tasks.get_task(task_id) print(task) In\u00a0[\u00a0]: Copied! <pre>metric = \"predictive_accuracy\"\nevals = openml.evaluations.list_evaluations(\n    function=metric, tasks=[task_id], output_format=\"dataframe\"\n)\n# Displaying the first 10 rows\nprint(evals.head(n=10))\n# Sorting the evaluations in decreasing order of the metric chosen\nevals = evals.sort_values(by=\"value\", ascending=False)\nprint(\"\\nDisplaying head of sorted dataframe: \")\nprint(evals.head())\n</pre> metric = \"predictive_accuracy\" evals = openml.evaluations.list_evaluations(     function=metric, tasks=[task_id], output_format=\"dataframe\" ) # Displaying the first 10 rows print(evals.head(n=10)) # Sorting the evaluations in decreasing order of the metric chosen evals = evals.sort_values(by=\"value\", ascending=False) print(\"\\nDisplaying head of sorted dataframe: \") print(evals.head()) In\u00a0[\u00a0]: Copied! <pre>from matplotlib import pyplot as plt\n\n\ndef plot_cdf(values, metric=\"predictive_accuracy\"):\n    max_val = max(values)\n    n, bins, patches = plt.hist(values, density=True, histtype=\"step\", cumulative=True, linewidth=3)\n    patches[0].set_xy(patches[0].get_xy()[:-1])\n    plt.xlim(max(0, min(values) - 0.1), 1)\n    plt.title(\"CDF\")\n    plt.xlabel(metric)\n    plt.ylabel(\"Likelihood\")\n    plt.grid(visible=True, which=\"major\", linestyle=\"-\")\n    plt.minorticks_on()\n    plt.grid(visible=True, which=\"minor\", linestyle=\"--\")\n    plt.axvline(max_val, linestyle=\"--\", color=\"gray\")\n    plt.text(max_val, 0, \"%.3f\" % max_val, fontsize=9)\n    plt.show()\n\n\nplot_cdf(evals.value, metric)\n</pre> from matplotlib import pyplot as plt   def plot_cdf(values, metric=\"predictive_accuracy\"):     max_val = max(values)     n, bins, patches = plt.hist(values, density=True, histtype=\"step\", cumulative=True, linewidth=3)     patches[0].set_xy(patches[0].get_xy()[:-1])     plt.xlim(max(0, min(values) - 0.1), 1)     plt.title(\"CDF\")     plt.xlabel(metric)     plt.ylabel(\"Likelihood\")     plt.grid(visible=True, which=\"major\", linestyle=\"-\")     plt.minorticks_on()     plt.grid(visible=True, which=\"minor\", linestyle=\"--\")     plt.axvline(max_val, linestyle=\"--\", color=\"gray\")     plt.text(max_val, 0, \"%.3f\" % max_val, fontsize=9)     plt.show()   plot_cdf(evals.value, metric) <p>This CDF plot shows that for the given task, based on the results of the runs uploaded, it is almost certain to achieve an accuracy above 52%, i.e., with non-zero probability. While the maximum accuracy seen till now is 96.5%.</p> In\u00a0[\u00a0]: Copied! <pre>import numpy as np\nimport pandas as pd\n\n\ndef plot_flow_compare(evaluations, top_n=10, metric=\"predictive_accuracy\"):\n    # Collecting the top 10 performing unique flow_id\n    flow_ids = evaluations.flow_id.unique()[:top_n]\n\n    df = pd.DataFrame()\n    # Creating a data frame containing only the metric values of the selected flows\n    #   assuming evaluations is sorted in decreasing order of metric\n    for i in range(len(flow_ids)):\n        flow_values = evaluations[evaluations.flow_id == flow_ids[i]].value\n        df = pd.concat([df, flow_values], ignore_index=True, axis=1)\n    fig, axs = plt.subplots()\n    df.boxplot()\n    axs.set_title(\"Boxplot comparing \" + metric + \" for different flows\")\n    axs.set_ylabel(metric)\n    axs.set_xlabel(\"Flow ID\")\n    axs.set_xticklabels(flow_ids)\n    axs.grid(which=\"major\", linestyle=\"-\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")\n    axs.minorticks_on()\n    axs.grid(which=\"minor\", linestyle=\"--\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")\n    # Counting the number of entries for each flow in the data frame\n    #   which gives the number of runs for each flow\n    flow_freq = list(df.count(axis=0, numeric_only=True))\n    for i in range(len(flow_ids)):\n        axs.text(i + 1.05, np.nanmin(df.values), str(flow_freq[i]) + \"\\nrun(s)\", fontsize=7)\n    plt.show()\n\n\nplot_flow_compare(evals, metric=metric, top_n=10)\n</pre> import numpy as np import pandas as pd   def plot_flow_compare(evaluations, top_n=10, metric=\"predictive_accuracy\"):     # Collecting the top 10 performing unique flow_id     flow_ids = evaluations.flow_id.unique()[:top_n]      df = pd.DataFrame()     # Creating a data frame containing only the metric values of the selected flows     #   assuming evaluations is sorted in decreasing order of metric     for i in range(len(flow_ids)):         flow_values = evaluations[evaluations.flow_id == flow_ids[i]].value         df = pd.concat([df, flow_values], ignore_index=True, axis=1)     fig, axs = plt.subplots()     df.boxplot()     axs.set_title(\"Boxplot comparing \" + metric + \" for different flows\")     axs.set_ylabel(metric)     axs.set_xlabel(\"Flow ID\")     axs.set_xticklabels(flow_ids)     axs.grid(which=\"major\", linestyle=\"-\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")     axs.minorticks_on()     axs.grid(which=\"minor\", linestyle=\"--\", linewidth=\"0.5\", color=\"gray\", axis=\"y\")     # Counting the number of entries for each flow in the data frame     #   which gives the number of runs for each flow     flow_freq = list(df.count(axis=0, numeric_only=True))     for i in range(len(flow_ids)):         axs.text(i + 1.05, np.nanmin(df.values), str(flow_freq[i]) + \"\\nrun(s)\", fontsize=7)     plt.show()   plot_flow_compare(evals, metric=metric, top_n=10) <p>The boxplots below show how the flows perform across multiple runs on the chosen task. The green horizontal lines represent the median accuracy of all the runs for that flow (number of runs denoted at the bottom of the boxplots). The higher the green line, the better the flow is for the task at hand. The ordering of the flows are in the descending order of the higest accuracy value seen under that flow.</p> <p>Printing the corresponding flow names for the top 10 performing flow IDs</p> In\u00a0[\u00a0]: Copied! <pre>top_n = 10\nflow_ids = evals.flow_id.unique()[:top_n]\nflow_names = evals.flow_name.unique()[:top_n]\nfor i in range(top_n):\n    print((flow_ids[i], flow_names[i]))\n</pre> top_n = 10 flow_ids = evals.flow_id.unique()[:top_n] flow_names = evals.flow_name.unique()[:top_n] for i in range(top_n):     print((flow_ids[i], flow_names[i])) In\u00a0[\u00a0]: Copied! <pre>evals_setups = openml.evaluations.list_evaluations_setups(\n    function=\"predictive_accuracy\", tasks=[31], size=100, sort_order=\"desc\"\n)\n\nprint(evals_setups.head())\n</pre> evals_setups = openml.evaluations.list_evaluations_setups(     function=\"predictive_accuracy\", tasks=[31], size=100, sort_order=\"desc\" )  print(evals_setups.head()) <p>Return evaluations for flow_id in descending order based on predictive_accuracy with hyperparameters. parameters_in_separate_columns returns parameters in separate columns</p> In\u00a0[\u00a0]: Copied! <pre>evals_setups = openml.evaluations.list_evaluations_setups(\n    function=\"predictive_accuracy\", flows=[6767], size=100, parameters_in_separate_columns=True\n)\n\nprint(evals_setups.head(10))\n\n# License: BSD 3-Clause\n</pre> evals_setups = openml.evaluations.list_evaluations_setups(     function=\"predictive_accuracy\", flows=[6767], size=100, parameters_in_separate_columns=True )  print(evals_setups.head(10))  # License: BSD 3-Clause"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#fetching-evaluations","title":"Fetching Evaluations\u00b6","text":"<p>Evaluations contain a concise summary of the results of all runs made. Each evaluation provides information on the dataset used, the flow applied, the setup used, the metric evaluated, and the result obtained on the metric, for each such run made. These collection of results can be used for efficient benchmarking of an algorithm and also allow transparent reuse of results from previous experiments on similar parameters.</p> <p>In this example, we shall do the following:</p> <ul> <li>Retrieve evaluations based on different metrics</li> <li>Fetch evaluations pertaining to a specific task</li> <li>Sort the obtained results in descending order of the metric</li> <li>Plot a cumulative distribution function for the evaluations</li> <li>Compare the top 10 performing flows based on the evaluation performance</li> <li>Retrieve evaluations with hyperparameter settings</li> </ul>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#listing-evaluations","title":"Listing evaluations\u00b6","text":"<p>Evaluations can be retrieved from the database in the chosen output format. Required filters can be applied to retrieve results from runs as required.</p> <p>We shall retrieve a small set (only 10 entries) to test the listing function for evaluations</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#viewing-a-sample-task","title":"Viewing a sample task\u00b6","text":"<p>Over here we shall briefly take a look at the details of the task. We will start by displaying a simple supervised classification task:</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#obtaining-all-the-evaluations-for-the-task","title":"Obtaining all the evaluations for the task\u00b6","text":"<p>We'll now obtain all the evaluations that were uploaded for the task we displayed previously. Note that we now filter the evaluations based on another parameter 'task'.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#obtaining-cdf-of-metric-for-chosen-task","title":"Obtaining CDF of metric for chosen task\u00b6","text":"<p>We shall now analyse how the performance of various flows have been on this task, by seeing the likelihood of the accuracy obtained across all runs. We shall now plot a cumulative distributive function (CDF) for the accuracies obtained.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#comparing-top-10-performing-flows","title":"Comparing top 10 performing flows\u00b6","text":"<p>Let us now try to see which flows generally performed the best for this task. For this, we shall compare the top performing flows.</p>"},{"location":"examples/30_extended/fetch_evaluations_tutorial/#obtaining-evaluations-with-hyperparameter-settings","title":"Obtaining evaluations with hyperparameter settings\u00b6","text":"<p>We'll now obtain the evaluations of a task and a flow with the hyperparameters</p> <p>List evaluations in descending order based on predictive_accuracy with hyperparameters</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/","title":"Preparing tasks and scikit-learn models","text":"<p>Measuring runtimes for Scikit-learn models</p> <p>The runtime of machine learning models on specific datasets can be a deciding factor on the choice of algorithms, especially for benchmarking and comparison purposes. OpenML's scikit-learn extension provides runtime data from runs of model fit and prediction on tasks or datasets, for both the CPU-clock as well as the actual wallclock-time incurred. The objective of this example is to illustrate how to retrieve such timing measures, and also offer some potential means of usage and interpretation of the same.</p> <p>It should be noted that there are multiple levels at which parallelism can occur.</p> <ul> <li><p>At the outermost level, OpenML tasks contain fixed data splits, on which the defined model/flow is executed. Thus, a model can be fit on each OpenML dataset fold in parallel using the <code>n_jobs</code> parameter to <code>run_model_on_task</code> or <code>run_flow_on_task</code> (illustrated under Case 2 &amp; 3 below).</p> </li> <li><p>The model/flow specified can also include scikit-learn models that perform their own parallelization. For instance, by specifying <code>n_jobs</code> in a Random Forest model definition (covered under Case 2 below).</p> </li> <li><p>The sklearn model can further be an HPO estimator and contain it's own parallelization. If the base estimator used also supports <code>parallelization</code>, then there's at least a 2-level nested definition for parallelization possible (covered under Case 3 below).</p> </li> </ul> <p>We shall cover these 5 representative scenarios for:</p> <ul> <li><p>(Case 1) Retrieving runtimes for Random Forest training and prediction on each of the cross-validation folds</p> </li> <li><p>(Case 2) Testing the above setting in a parallel setup and monitor the difference using runtimes retrieved</p> </li> <li><p>(Case 3) Comparing RandomSearchCV and GridSearchCV on the above task based on runtimes</p> </li> <li><p>(Case 4) Running models that don't run in parallel or models which scikit-learn doesn't parallelize</p> </li> <li><p>(Case 5) Running models that do not release the Python Global Interpreter Lock (GIL)</p> </li> </ul> <p>import openml import numpy as np from matplotlib import pyplot as plt from joblib.parallel import parallel_backend</p> <p>from sklearn.naive_bayes import GaussianNB from sklearn.tree import DecisionTreeClassifier from sklearn.neural_network import MLPClassifier from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV, RandomizedSearchCV</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 167119\n\ntask = openml.tasks.get_task(task_id)\nprint(task)\n\n# Viewing associated data\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n\n\n# Creating utility function\ndef print_compare_runtimes(measures):\n    for repeat, val1 in measures[\"usercpu_time_millis_training\"].items():\n        for fold, val2 in val1.items():\n            print(\n                \"Repeat #{}-Fold #{}: CPU-{:.3f} vs Wall-{:.3f}\".format(\n                    repeat, fold, val2, measures[\"wall_clock_time_millis_training\"][repeat][fold]\n                )\n            )\n</pre> task_id = 167119  task = openml.tasks.get_task(task_id) print(task)  # Viewing associated data n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) )   # Creating utility function def print_compare_runtimes(measures):     for repeat, val1 in measures[\"usercpu_time_millis_training\"].items():         for fold, val2 in val1.items():             print(                 \"Repeat #{}-Fold #{}: CPU-{:.3f} vs Wall-{:.3f}\".format(                     repeat, fold, val2, measures[\"wall_clock_time_millis_training\"][repeat][fold]                 )             ) In\u00a0[\u00a0]: Copied! <pre>clf = RandomForestClassifier(n_estimators=10)\n\nrun1 = openml.runs.run_model_on_task(\n    model=clf,\n    task=task,\n    upload_flow=False,\n    avoid_duplicate_runs=False,\n)\nmeasures = run1.fold_evaluations\n\nprint(\"The timing and performance metrics available: \")\nfor key in measures.keys():\n    print(key)\nprint()\n\nprint(\n    \"The performance metric is recorded under `predictive_accuracy` per \"\n    \"fold and can be retrieved as: \"\n)\nfor repeat, val1 in measures[\"predictive_accuracy\"].items():\n    for fold, val2 in val1.items():\n        print(\"Repeat #{}-Fold #{}: {:.4f}\".format(repeat, fold, val2))\n    print()\n</pre> clf = RandomForestClassifier(n_estimators=10)  run1 = openml.runs.run_model_on_task(     model=clf,     task=task,     upload_flow=False,     avoid_duplicate_runs=False, ) measures = run1.fold_evaluations  print(\"The timing and performance metrics available: \") for key in measures.keys():     print(key) print()  print(     \"The performance metric is recorded under `predictive_accuracy` per \"     \"fold and can be retrieved as: \" ) for repeat, val1 in measures[\"predictive_accuracy\"].items():     for fold, val2 in val1.items():         print(\"Repeat #{}-Fold #{}: {:.4f}\".format(repeat, fold, val2))     print() <p>The remaining entries recorded in <code>measures</code> are the runtime records related as:</p> <p>usercpu_time_millis = usercpu_time_millis_training + usercpu_time_millis_testing</p> <p>wall_clock_time_millis = wall_clock_time_millis_training + wall_clock_time_millis_testing</p> <p>The timing measures recorded as <code>*_millis_training</code> contain the per repeat-per fold timing incurred for the execution of the <code>.fit()</code> procedure of the model. For <code>usercpu_time_*</code> the time recorded using <code>time.process_time()</code> is converted to <code>milliseconds</code> and stored. Similarly, <code>time.time()</code> is used to record the time entry for <code>wall_clock_time_*</code>. The <code>*_millis_testing</code> entry follows the same procedure but for time taken for the <code>.predict()</code> procedure.</p> <p>Comparing the CPU and wall-clock training times of the Random Forest model</p> In\u00a0[\u00a0]: Copied! <pre>print_compare_runtimes(measures)\n</pre> print_compare_runtimes(measures) In\u00a0[\u00a0]: Copied! <pre>clf = RandomForestClassifier(n_estimators=10, n_jobs=2)\n\nrun2 = openml.runs.run_model_on_task(\n    model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n)\nmeasures = run2.fold_evaluations\n# The wall-clock time recorded per fold should be lesser than Case 1 above\nprint_compare_runtimes(measures)\n</pre> clf = RandomForestClassifier(n_estimators=10, n_jobs=2)  run2 = openml.runs.run_model_on_task(     model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False ) measures = run2.fold_evaluations # The wall-clock time recorded per fold should be lesser than Case 1 above print_compare_runtimes(measures) <p>Running a Random Forest model on an OpenML task in parallel (all cores available):</p> In\u00a0[\u00a0]: Copied! <pre># Redefining the model to use all available cores with `n_jobs=-1`\nclf = RandomForestClassifier(n_estimators=10, n_jobs=-1)\n\nrun3 = openml.runs.run_model_on_task(\n    model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n)\nmeasures = run3.fold_evaluations\n</pre> # Redefining the model to use all available cores with `n_jobs=-1` clf = RandomForestClassifier(n_estimators=10, n_jobs=-1)  run3 = openml.runs.run_model_on_task(     model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False ) measures = run3.fold_evaluations <p>The wall-clock time recorded per fold should be lesser than the case above, if more than 2 CPU cores are available. The speed-up is more pronounced for larger datasets. print_compare_runtimes(measures)</p> <p>We can now observe that the ratio of CPU time to wallclock time is lower than in case 1. This happens because joblib by default spawns subprocesses for the workloads for which CPU time cannot be tracked. Therefore, interpreting the reported CPU and wallclock time requires knowledge of the parallelization applied at runtime.</p> <p>Running the same task with a different parallel backend. Joblib provides multiple backends: {<code>loky</code> (default), <code>multiprocessing</code>, <code>dask</code>, <code>threading</code>, <code>sequential</code>}. The backend can be explicitly set using a joblib context manager. The behaviour of the job distribution can change and therefore the scale of runtimes recorded too.</p> In\u00a0[\u00a0]: Copied! <pre>with parallel_backend(backend=\"multiprocessing\", n_jobs=-1):\n    run3_ = openml.runs.run_model_on_task(\n        model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n    )\nmeasures = run3_.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> with parallel_backend(backend=\"multiprocessing\", n_jobs=-1):     run3_ = openml.runs.run_model_on_task(         model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False     ) measures = run3_.fold_evaluations print_compare_runtimes(measures) <p>The CPU time interpretation becomes ambiguous when jobs are distributed over an unknown number of cores or when subprocesses are spawned for which the CPU time cannot be tracked, as in the examples above. It is impossible for OpenML-Python to capture the availability of the number of cores/threads, their eventual utilisation and whether workloads are executed in subprocesses, for various cases that can arise as demonstrated in the rest of the example. Therefore, the final interpretation of the runtimes is left to the <code>user</code>.</p> In\u00a0[\u00a0]: Copied! <pre>from sklearn.model_selection import GridSearchCV\n\nclf = RandomForestClassifier(n_estimators=10, n_jobs=2)\n\n# GridSearchCV model\nn_iter = 5\ngrid_pipe = GridSearchCV(\n    estimator=clf,\n    param_grid={\"n_estimators\": np.linspace(start=1, stop=50, num=n_iter).astype(int).tolist()},\n    cv=2,\n    n_jobs=2,\n)\n\nrun4 = openml.runs.run_model_on_task(\n    model=grid_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2\n)\nmeasures = run4.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> from sklearn.model_selection import GridSearchCV  clf = RandomForestClassifier(n_estimators=10, n_jobs=2)  # GridSearchCV model n_iter = 5 grid_pipe = GridSearchCV(     estimator=clf,     param_grid={\"n_estimators\": np.linspace(start=1, stop=50, num=n_iter).astype(int).tolist()},     cv=2,     n_jobs=2, )  run4 = openml.runs.run_model_on_task(     model=grid_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2 ) measures = run4.fold_evaluations print_compare_runtimes(measures) <p>Like any optimisation problem, scikit-learn's HPO estimators also generate a sequence of configurations which are evaluated, using which the best found configuration is tracked throughout the trace. The OpenML run object stores these traces as OpenMLRunTrace objects accessible using keys of the pattern (repeat, fold, iterations). Here <code>fold</code> implies the outer-cross validation fold as obtained from the task data splits in OpenML. GridSearchCV here performs grid search over the inner-cross validation folds as parameterized by the <code>cv</code> parameter. Since <code>GridSearchCV</code> in this example performs a <code>2-fold</code> cross validation, the runtime recorded per repeat-per fold in the run object is for the entire <code>fit()</code> procedure of GridSearchCV thus subsuming the runtimes of the 2-fold (inner) CV search performed.</p> In\u00a0[\u00a0]: Copied! <pre># We earlier extracted the number of repeats and folds for this task:\nprint(\"# repeats: {}\\n# folds: {}\".format(n_repeats, n_folds))\n\n# To extract the training runtime of the first repeat, first fold:\nprint(run4.fold_evaluations[\"wall_clock_time_millis_training\"][0][0])\n</pre> # We earlier extracted the number of repeats and folds for this task: print(\"# repeats: {}\\n# folds: {}\".format(n_repeats, n_folds))  # To extract the training runtime of the first repeat, first fold: print(run4.fold_evaluations[\"wall_clock_time_millis_training\"][0][0]) <p>To extract the training runtime of the 1-st repeat, 4-th (outer) fold and also to fetch the parameters and performance of the evaluations made during the 1-st repeat, 4-th fold evaluation by the Grid Search model.</p> In\u00a0[\u00a0]: Copied! <pre>_repeat = 0\n_fold = 3\nprint(\n    \"Total runtime for repeat {}'s fold {}: {:4f} ms\".format(\n        _repeat, _fold, run4.fold_evaluations[\"wall_clock_time_millis_training\"][_repeat][_fold]\n    )\n)\nfor i in range(n_iter):\n    key = (_repeat, _fold, i)\n    r = run4.trace.trace_iterations[key]\n    print(\n        \"n_estimators: {:&gt;2} - score: {:.3f}\".format(\n            r.parameters[\"parameter_n_estimators\"], r.evaluation\n        )\n    )\n</pre> _repeat = 0 _fold = 3 print(     \"Total runtime for repeat {}'s fold {}: {:4f} ms\".format(         _repeat, _fold, run4.fold_evaluations[\"wall_clock_time_millis_training\"][_repeat][_fold]     ) ) for i in range(n_iter):     key = (_repeat, _fold, i)     r = run4.trace.trace_iterations[key]     print(         \"n_estimators: {:&gt;2} - score: {:.3f}\".format(             r.parameters[\"parameter_n_estimators\"], r.evaluation         )     ) <p>Scikit-learn's HPO estimators also come with an argument <code>refit=True</code> as a default. In our previous model definition it was set to True by default, which meant that the best found hyperparameter configuration was used to refit or retrain the model without any inner cross validation. This extra refit time measure is provided by the scikit-learn model as the attribute <code>refit_time_</code>. This time is included in the <code>wall_clock_time_millis_training</code> measure.</p> <p>For non-HPO estimators, <code>wall_clock_time_millis = wall_clock_time_millis_training + wall_clock_time_millis_testing</code>.</p> <p>For HPO estimators, <code>wall_clock_time_millis = wall_clock_time_millis_training + wall_clock_time_millis_testing + refit_time</code>.</p> <p>This refit time can therefore be explicitly extracted in this manner:</p> In\u00a0[\u00a0]: Copied! <pre>def extract_refit_time(run, repeat, fold):\n    refit_time = (\n        run.fold_evaluations[\"wall_clock_time_millis\"][repeat][fold]\n        - run.fold_evaluations[\"wall_clock_time_millis_training\"][repeat][fold]\n        - run.fold_evaluations[\"wall_clock_time_millis_testing\"][repeat][fold]\n    )\n    return refit_time\n\n\nfor repeat in range(n_repeats):\n    for fold in range(n_folds):\n        print(\n            \"Repeat #{}-Fold #{}: {:.4f}\".format(\n                repeat, fold, extract_refit_time(run4, repeat, fold)\n            )\n        )\n</pre>  def extract_refit_time(run, repeat, fold):     refit_time = (         run.fold_evaluations[\"wall_clock_time_millis\"][repeat][fold]         - run.fold_evaluations[\"wall_clock_time_millis_training\"][repeat][fold]         - run.fold_evaluations[\"wall_clock_time_millis_testing\"][repeat][fold]     )     return refit_time   for repeat in range(n_repeats):     for fold in range(n_folds):         print(             \"Repeat #{}-Fold #{}: {:.4f}\".format(                 repeat, fold, extract_refit_time(run4, repeat, fold)             )         ) <p>Along with the GridSearchCV already used above, we demonstrate how such optimisation traces can be retrieved by showing an application of these traces - comparing the speed of finding the best configuration using RandomizedSearchCV and GridSearchCV available with scikit-learn.</p> In\u00a0[\u00a0]: Copied! <pre># RandomizedSearchCV model\nrs_pipe = RandomizedSearchCV(\n    estimator=clf,\n    param_distributions={\n        \"n_estimators\": np.linspace(start=1, stop=50, num=15).astype(int).tolist()\n    },\n    cv=2,\n    n_iter=n_iter,\n    n_jobs=2,\n)\nrun5 = openml.runs.run_model_on_task(\n    model=rs_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2\n)\n</pre> # RandomizedSearchCV model rs_pipe = RandomizedSearchCV(     estimator=clf,     param_distributions={         \"n_estimators\": np.linspace(start=1, stop=50, num=15).astype(int).tolist()     },     cv=2,     n_iter=n_iter,     n_jobs=2, ) run5 = openml.runs.run_model_on_task(     model=rs_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2 ) <p>Since for the call to <code>openml.runs.run_model_on_task</code> the parameter <code>n_jobs</code> is set to its default <code>None</code>, the evaluations across the OpenML folds are not parallelized. Hence, the time recorded is agnostic to the <code>n_jobs</code> being set at both the HPO estimator <code>GridSearchCV</code> as well as the base estimator <code>RandomForestClassifier</code> in this case. The OpenML extension only records the time taken for the completion of the complete <code>fit()</code> call, per-repeat per-fold.</p> <p>This notion can be used to extract and plot the best found performance per fold by the HPO model and the corresponding time taken for search across that fold. Moreover, since <code>n_jobs=None</code> for <code>openml.runs.run_model_on_task</code> the runtimes per fold can be cumulatively added to plot the trace against time.</p> In\u00a0[\u00a0]: Copied! <pre>def extract_trace_data(run, n_repeats, n_folds, n_iter, key=None):\n    key = \"wall_clock_time_millis_training\" if key is None else key\n    data = {\"score\": [], \"runtime\": []}\n    for i_r in range(n_repeats):\n        for i_f in range(n_folds):\n            data[\"runtime\"].append(run.fold_evaluations[key][i_r][i_f])\n            for i_i in range(n_iter):\n                r = run.trace.trace_iterations[(i_r, i_f, i_i)]\n                if r.selected:\n                    data[\"score\"].append(r.evaluation)\n                    break\n    return data\n\n\ndef get_incumbent_trace(trace):\n    best_score = 1\n    inc_trace = []\n    for i, r in enumerate(trace):\n        if i == 0 or (1 - r) &lt; best_score:\n            best_score = 1 - r\n        inc_trace.append(best_score)\n    return inc_trace\n\n\ngrid_data = extract_trace_data(run4, n_repeats, n_folds, n_iter)\nrs_data = extract_trace_data(run5, n_repeats, n_folds, n_iter)\n\nplt.clf()\nplt.plot(\n    np.cumsum(grid_data[\"runtime\"]), get_incumbent_trace(grid_data[\"score\"]), label=\"Grid Search\"\n)\nplt.plot(\n    np.cumsum(rs_data[\"runtime\"]), get_incumbent_trace(rs_data[\"score\"]), label=\"Random Search\"\n)\nplt.xscale(\"log\")\nplt.yscale(\"log\")\nplt.xlabel(\"Wallclock time (in milliseconds)\")\nplt.ylabel(\"1 - Accuracy\")\nplt.title(\"Optimisation Trace Comparison\")\nplt.legend()\nplt.show()\n</pre> def extract_trace_data(run, n_repeats, n_folds, n_iter, key=None):     key = \"wall_clock_time_millis_training\" if key is None else key     data = {\"score\": [], \"runtime\": []}     for i_r in range(n_repeats):         for i_f in range(n_folds):             data[\"runtime\"].append(run.fold_evaluations[key][i_r][i_f])             for i_i in range(n_iter):                 r = run.trace.trace_iterations[(i_r, i_f, i_i)]                 if r.selected:                     data[\"score\"].append(r.evaluation)                     break     return data   def get_incumbent_trace(trace):     best_score = 1     inc_trace = []     for i, r in enumerate(trace):         if i == 0 or (1 - r) &lt; best_score:             best_score = 1 - r         inc_trace.append(best_score)     return inc_trace   grid_data = extract_trace_data(run4, n_repeats, n_folds, n_iter) rs_data = extract_trace_data(run5, n_repeats, n_folds, n_iter)  plt.clf() plt.plot(     np.cumsum(grid_data[\"runtime\"]), get_incumbent_trace(grid_data[\"score\"]), label=\"Grid Search\" ) plt.plot(     np.cumsum(rs_data[\"runtime\"]), get_incumbent_trace(rs_data[\"score\"]), label=\"Random Search\" ) plt.xscale(\"log\") plt.yscale(\"log\") plt.xlabel(\"Wallclock time (in milliseconds)\") plt.ylabel(\"1 - Accuracy\") plt.title(\"Optimisation Trace Comparison\") plt.legend() plt.show() In\u00a0[\u00a0]: Copied! <pre>dt = DecisionTreeClassifier()\n\nrun6 = openml.runs.run_model_on_task(\n    model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2\n)\nmeasures = run6.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> dt = DecisionTreeClassifier()  run6 = openml.runs.run_model_on_task(     model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2 ) measures = run6.fold_evaluations print_compare_runtimes(measures) <p>Although the decision tree does not run in parallel, it can release the <code>Python GIL &lt;https://docs.python.org/dev/glossary.html#term-global-interpreter-lock&gt;</code>_. This can result in surprising runtime measures as demonstrated below:</p> In\u00a0[\u00a0]: Copied! <pre>with parallel_backend(\"threading\", n_jobs=-1):\n    run7 = openml.runs.run_model_on_task(\n        model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False\n    )\nmeasures = run7.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> with parallel_backend(\"threading\", n_jobs=-1):     run7 = openml.runs.run_model_on_task(         model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False     ) measures = run7.fold_evaluations print_compare_runtimes(measures) <p>Running a Neural Network from scikit-learn that uses scikit-learn independent parallelism using libraries such as MKL, OpenBLAS or BLIS.</p> In\u00a0[\u00a0]: Copied! <pre>mlp = MLPClassifier(max_iter=10)\n\nrun8 = openml.runs.run_model_on_task(\n    model=mlp, task=task, upload_flow=False, avoid_duplicate_runs=False\n)\nmeasures = run8.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> mlp = MLPClassifier(max_iter=10)  run8 = openml.runs.run_model_on_task(     model=mlp, task=task, upload_flow=False, avoid_duplicate_runs=False ) measures = run8.fold_evaluations print_compare_runtimes(measures) In\u00a0[\u00a0]: Copied! <pre>clf = GaussianNB()\n\nwith parallel_backend(\"multiprocessing\", n_jobs=-1):\n    run9 = openml.runs.run_model_on_task(\n        model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False\n    )\nmeasures = run9.fold_evaluations\nprint_compare_runtimes(measures)\n</pre> clf = GaussianNB()  with parallel_backend(\"multiprocessing\", n_jobs=-1):     run9 = openml.runs.run_model_on_task(         model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False     ) measures = run9.fold_evaluations print_compare_runtimes(measures)"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#preparing-tasks-and-scikit-learn-models","title":"Preparing tasks and scikit-learn models\u00b6","text":""},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-1-running-a-random-forest-model-on-an-openml-task","title":"Case 1: Running a Random Forest model on an OpenML task\u00b6","text":"<p>We'll run a Random Forest model and obtain an OpenML run object. We can see the evaluations recorded per fold for the dataset and the information available for this run.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-2-running-scikit-learn-model-on-an-openml-task-in-parallel","title":"Case 2: Running Scikit-learn model on an OpenML task in parallel\u00b6","text":"<p>Redefining the model to allow parallelism with <code>n_jobs=2</code> (2 cores)</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-3-running-and-benchmarking-hpo-algorithms-with-their-runtimes","title":"Case 3: Running and benchmarking HPO algorithms with their runtimes\u00b6","text":"<p>We shall now optimize a similar RandomForest model for the same task using scikit-learn's HPO support by using GridSearchCV to optimize our earlier RandomForest model's hyperparameter <code>n_estimators</code>. Scikit-learn also provides a <code>refit_time_</code> for such HPO models, i.e., the time incurred by training and evaluating the model on the best found parameter setting. This is included in the <code>wall_clock_time_millis_training</code> measure recorded.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-4-running-models-that-scikit-learn-doesnt-parallelize","title":"Case 4: Running models that scikit-learn doesn't parallelize\u00b6","text":"<p>Both scikit-learn and OpenML depend on parallelism implemented through <code>joblib</code>. However, there can be cases where either models cannot be parallelized or don't depend on joblib for its parallelism. 2 such cases are illustrated below.</p> <p>Running a Decision Tree model that doesn't support parallelism implicitly, but using OpenML to parallelize evaluations for the outer-cross validation folds.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#case-5-running-scikit-learn-models-that-dont-release-gil","title":"Case 5: Running Scikit-learn models that don't release GIL\u00b6","text":"<p>Certain Scikit-learn models do not release the Python GIL and are also not executed in parallel via a BLAS library. In such cases, the CPU times and wallclock times are most likely trustworthy. Note however that only very few models such as naive Bayes models are of this kind.</p>"},{"location":"examples/30_extended/fetch_runtimes_tutorial/#summmary","title":"Summmary\u00b6","text":"<p>The scikit-learn extension for OpenML-Python records model runtimes for the CPU-clock and the wall-clock times. The above examples illustrated how these recorded runtimes can be extracted when using a scikit-learn model and under parallel setups too. To summarize, the scikit-learn extension measures the:</p> <ul> <li><p><code>CPU-time</code> &amp; <code>wallclock-time</code> for the whole run</p> <ul> <li>A run here corresponds to a call to <code>run_model_on_task</code> or <code>run_flow_on_task</code></li> <li>The recorded time is for the model fit for each of the outer-cross validations folds, i.e., the OpenML data splits</li> </ul> </li> <li><p>Python's <code>time</code> module is used to compute the runtimes</p> <ul> <li><code>CPU-time</code> is recorded using the responses of <code>time.process_time()</code></li> <li><code>wallclock-time</code> is recorded using the responses of <code>time.time()</code></li> </ul> </li> <li><p>The timings recorded by OpenML per outer-cross validation fold is agnostic to model parallelisation</p> <ul> <li>The wallclock times reported in Case 2 above highlights the speed-up on using <code>n_jobs=-1</code> in comparison to <code>n_jobs=2</code>, since the timing recorded by OpenML is for the entire <code>fit()</code> procedure, whereas the parallelisation is performed inside <code>fit()</code> by scikit-learn</li> <li>The CPU-time for models that are run in parallel can be difficult to interpret</li> </ul> </li> <li><p><code>CPU-time</code> &amp; <code>wallclock-time</code> for each search per outer fold in an HPO run</p> <ul> <li>Reports the total time for performing search on each of the OpenML data split, subsuming any sort of parallelism that happened as part of the HPO estimator or the underlying base estimator</li> <li>Also allows extraction of the <code>refit_time</code> that scikit-learn measures using <code>time.time()</code> for retraining the model per outer fold, for the best found configuration</li> </ul> </li> <li><p><code>CPU-time</code> &amp; <code>wallclock-time</code> for models that scikit-learn doesn't parallelize</p> <ul> <li>Models like Decision Trees or naive Bayes don't parallelize and thus both the wallclock and CPU times are similar in runtime for the OpenML call</li> <li>However, models implemented in Cython, such as the Decision Trees can release the GIL and still run in parallel if a <code>threading</code> backend is used by joblib.</li> <li>Scikit-learn Neural Networks can undergo parallelization implicitly owing to thread-level parallelism involved in the linear algebraic operations and thus the wallclock-time and CPU-time can differ.</li> </ul> </li> </ul> <p>Because of all the cases mentioned above it is crucial to understand which case is triggered when reporting runtimes for scikit-learn models measured with OpenML-Python! License: BSD 3-Clause</p>"},{"location":"examples/30_extended/flow_id_tutorial/","title":"Obtaining Flow IDs","text":"In\u00a0[\u00a0]: Copied! <pre>import sklearn.tree\n\nimport openml\n</pre> import sklearn.tree  import openml <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre># Defining a classifier\nclf = sklearn.tree.DecisionTreeClassifier()\n</pre> # Defining a classifier clf = sklearn.tree.DecisionTreeClassifier() In\u00a0[\u00a0]: Copied! <pre>flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish()\nflow_id = flow.flow_id\nprint(flow_id)\n</pre> flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish() flow_id = flow.flow_id print(flow_id) <p>This piece of code is rather involved. First, it retrieves a :class:<code>~openml.extensions.Extension</code> which is registered and can handle the given model, in our case it is :class:<code>openml.extensions.sklearn.SklearnExtension</code>. Second, the extension converts the classifier into an instance of :class:<code>openml.OpenMLFlow</code>. Third and finally, the publish method checks whether the current flow is already present on OpenML. If not, it uploads the flow, otherwise, it updates the current instance with all information computed by the server (which is obviously also done when uploading/publishing a flow).</p> <p>To simplify the usage we have created a helper function which automates all these steps:</p> In\u00a0[\u00a0]: Copied! <pre>flow_id = openml.flows.get_flow_id(model=clf)\nprint(flow_id)\n</pre> flow_id = openml.flows.get_flow_id(model=clf) print(flow_id) In\u00a0[\u00a0]: Copied! <pre>print(flow.name, flow.external_version)\n</pre> print(flow.name, flow.external_version) <p>The name and external version are automatically added to a flow when constructing it from a model. We can then use them to retrieve the flow id as follows:</p> In\u00a0[\u00a0]: Copied! <pre>flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version)\nprint(flow_id)\n</pre> flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version) print(flow_id) <p>We can also retrieve all flows for a given name:</p> In\u00a0[\u00a0]: Copied! <pre>flow_ids = openml.flows.get_flow_id(name=flow.name)\nprint(flow_ids)\n</pre> flow_ids = openml.flows.get_flow_id(name=flow.name) print(flow_ids) <p>This also works with the actual model (generalizing the first part of this example):</p> In\u00a0[\u00a0]: Copied! <pre>flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False)\nprint(flow_ids)\n</pre> flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False) print(flow_ids) In\u00a0[\u00a0]: Copied! <pre># Deactivating test configuration\nopenml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> # Deactivating test configuration openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/flow_id_tutorial/#obtaining-flow-ids","title":"Obtaining Flow IDs\u00b6","text":"<p>This tutorial discusses different ways to obtain the ID of a flow in order to perform further analysis.</p>"},{"location":"examples/30_extended/flow_id_tutorial/#1-obtaining-a-flow-given-a-classifier","title":"1. Obtaining a flow given a classifier\u00b6","text":""},{"location":"examples/30_extended/flow_id_tutorial/#2-obtaining-a-flow-given-its-name","title":"2. Obtaining a flow given its name\u00b6","text":"<p>The schema of a flow is given in XSD ( here).  # noqa E501 Only two fields are required, a unique name, and an external version. While it should be pretty obvious why we need a name, the need for the additional external version information might not be immediately clear. However, this information is very important as it allows to have multiple flows with the same name for different versions of a software. This might be necessary if an algorithm or implementation introduces, renames or drop hyperparameters over time.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/","title":"Flows and Runs","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree\n</pre> import openml from sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre># NOTE: We are using dataset 68 from the test server: https://test.openml.org/d/68\ndataset = openml.datasets.get_dataset(68)\nX, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nclf = neighbors.KNeighborsClassifier(n_neighbors=1)\nclf.fit(X, y)\n</pre> # NOTE: We are using dataset 68 from the test server: https://test.openml.org/d/68 dataset = openml.datasets.get_dataset(68) X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) clf = neighbors.KNeighborsClassifier(n_neighbors=1) clf.fit(X, y) <p>You can also ask for meta-data to automatically preprocess the data.</p> <ul> <li>e.g. categorical features -&gt; do feature encoding</li> </ul> In\u00a0[\u00a0]: Copied! <pre>dataset = openml.datasets.get_dataset(17)\nX, y, categorical_indicator, attribute_names = dataset.get_data(\n    target=dataset.default_target_attribute\n)\nprint(f\"Categorical features: {categorical_indicator}\")\ntransformer = compose.ColumnTransformer(\n    [(\"one_hot_encoder\", preprocessing.OneHotEncoder(categories=\"auto\"), categorical_indicator)]\n)\nX = transformer.fit_transform(X)\nclf.fit(X, y)\n</pre> dataset = openml.datasets.get_dataset(17) X, y, categorical_indicator, attribute_names = dataset.get_data(     target=dataset.default_target_attribute ) print(f\"Categorical features: {categorical_indicator}\") transformer = compose.ColumnTransformer(     [(\"one_hot_encoder\", preprocessing.OneHotEncoder(categories=\"auto\"), categorical_indicator)] ) X = transformer.fit_transform(X) clf.fit(X, y) In\u00a0[\u00a0]: Copied! <pre># Get a task\ntask = openml.tasks.get_task(403)\n\n# Build any classifier or pipeline\nclf = tree.DecisionTreeClassifier()\n\n# Run the flow\nrun = openml.runs.run_model_on_task(clf, task)\n\nprint(run)\n</pre> # Get a task task = openml.tasks.get_task(403)  # Build any classifier or pipeline clf = tree.DecisionTreeClassifier()  # Run the flow run = openml.runs.run_model_on_task(clf, task)  print(run) <p>Share the run on the OpenML server</p> <p>So far the run is only available locally. By calling the publish function, the run is sent to the OpenML server:</p> In\u00a0[\u00a0]: Copied! <pre>myrun = run.publish()\n# For this tutorial, our configuration publishes to the test server\n# as to not pollute the main server.\nprint(f\"Uploaded to {myrun.openml_url}\")\n</pre> myrun = run.publish() # For this tutorial, our configuration publishes to the test server # as to not pollute the main server. print(f\"Uploaded to {myrun.openml_url}\") <p>We can now also inspect the flow object which was automatically created:</p> In\u00a0[\u00a0]: Copied! <pre>flow = openml.flows.get_flow(run.flow_id)\nprint(flow)\n</pre> flow = openml.flows.get_flow(run.flow_id) print(flow) In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(96)\n\n# OpenML helper functions for sklearn can be plugged in directly for complicated pipelines\nfrom openml.extensions.sklearn import cat, cont\n\npipe = pipeline.Pipeline(\n    steps=[\n        (\n            \"Preprocessing\",\n            compose.ColumnTransformer(\n                [\n                    (\n                        \"categorical\",\n                        preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),\n                        cat,  # returns the categorical feature indices\n                    ),\n                    (\n                        \"continuous\",\n                        impute.SimpleImputer(strategy=\"median\"),\n                        cont,\n                    ),  # returns the numeric feature indices\n                ]\n            ),\n        ),\n        (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),\n    ]\n)\n\nrun = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)\nmyrun = run.publish()\nprint(f\"Uploaded to {myrun.openml_url}\")\n</pre> task = openml.tasks.get_task(96)  # OpenML helper functions for sklearn can be plugged in directly for complicated pipelines from openml.extensions.sklearn import cat, cont  pipe = pipeline.Pipeline(     steps=[         (             \"Preprocessing\",             compose.ColumnTransformer(                 [                     (                         \"categorical\",                         preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),                         cat,  # returns the categorical feature indices                     ),                     (                         \"continuous\",                         impute.SimpleImputer(strategy=\"median\"),                         cont,                     ),  # returns the numeric feature indices                 ]             ),         ),         (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),     ] )  run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False) myrun = run.publish() print(f\"Uploaded to {myrun.openml_url}\") <p>The above pipeline works with the helper functions that internally deal with pandas DataFrame. In the case, pandas is not available, or a NumPy based data processing is the requirement, the above pipeline is presented below to work with NumPy.</p> In\u00a0[\u00a0]: Copied! <pre># Extracting the indices of the categorical columns\nfeatures = task.get_dataset().features\ncategorical_feature_indices = []\nnumeric_feature_indices = []\nfor i in range(len(features)):\n    if features[i].name == task.target_name:\n        continue\n    if features[i].data_type == \"nominal\":\n        categorical_feature_indices.append(i)\n    else:\n        numeric_feature_indices.append(i)\n\npipe = pipeline.Pipeline(\n    steps=[\n        (\n            \"Preprocessing\",\n            compose.ColumnTransformer(\n                [\n                    (\n                        \"categorical\",\n                        preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),\n                        categorical_feature_indices,\n                    ),\n                    (\n                        \"continuous\",\n                        impute.SimpleImputer(strategy=\"median\"),\n                        numeric_feature_indices,\n                    ),\n                ]\n            ),\n        ),\n        (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),\n    ]\n)\n\nrun = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)\nmyrun = run.publish()\nprint(f\"Uploaded to {myrun.openml_url}\")\n</pre> # Extracting the indices of the categorical columns features = task.get_dataset().features categorical_feature_indices = [] numeric_feature_indices = [] for i in range(len(features)):     if features[i].name == task.target_name:         continue     if features[i].data_type == \"nominal\":         categorical_feature_indices.append(i)     else:         numeric_feature_indices.append(i)  pipe = pipeline.Pipeline(     steps=[         (             \"Preprocessing\",             compose.ColumnTransformer(                 [                     (                         \"categorical\",                         preprocessing.OneHotEncoder(sparse=False, handle_unknown=\"ignore\"),                         categorical_feature_indices,                     ),                     (                         \"continuous\",                         impute.SimpleImputer(strategy=\"median\"),                         numeric_feature_indices,                     ),                 ]             ),         ),         (\"Classifier\", ensemble.RandomForestClassifier(n_estimators=10)),     ] )  run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False) myrun = run.publish() print(f\"Uploaded to {myrun.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(96)\n\n# The following lines can then be executed offline:\nrun = openml.runs.run_model_on_task(\n    pipe,\n    task,\n    avoid_duplicate_runs=False,\n    upload_flow=False,\n)\n\n# The run may be stored offline, and the flow will be stored along with it:\nrun.to_filesystem(directory=\"myrun\")\n\n# They may be loaded and uploaded at a later time\nrun = openml.runs.OpenMLRun.from_filesystem(directory=\"myrun\")\nrun.publish()\n\n# Publishing the run will automatically upload the related flow if\n# it does not yet exist on the server.\n</pre> task = openml.tasks.get_task(96)  # The following lines can then be executed offline: run = openml.runs.run_model_on_task(     pipe,     task,     avoid_duplicate_runs=False,     upload_flow=False, )  # The run may be stored offline, and the flow will be stored along with it: run.to_filesystem(directory=\"myrun\")  # They may be loaded and uploaded at a later time run = openml.runs.OpenMLRun.from_filesystem(directory=\"myrun\") run.publish()  # Publishing the run will automatically upload the related flow if # it does not yet exist on the server. <p>Alternatively, one can also directly run flows.</p> In\u00a0[\u00a0]: Copied! <pre># Get a task\ntask = openml.tasks.get_task(403)\n\n# Build any classifier or pipeline\nclf = tree.ExtraTreeClassifier()\n\n# Obtain the scikit-learn extension interface to convert the classifier\n# into a flow object.\nextension = openml.extensions.get_extension_by_model(clf)\nflow = extension.model_to_flow(clf)\n\nrun = openml.runs.run_flow_on_task(flow, task)\n</pre> # Get a task task = openml.tasks.get_task(403)  # Build any classifier or pipeline clf = tree.ExtraTreeClassifier()  # Obtain the scikit-learn extension interface to convert the classifier # into a flow object. extension = openml.extensions.get_extension_by_model(clf) flow = extension.model_to_flow(clf)  run = openml.runs.run_flow_on_task(flow, task) In\u00a0[\u00a0]: Copied! <pre># Easy benchmarking:\nfor task_id in [115]:  # Add further tasks. Disclaimer: they might take some time\n    task = openml.tasks.get_task(task_id)\n    data = openml.datasets.get_dataset(task.dataset_id)\n    clf = neighbors.KNeighborsClassifier(n_neighbors=5)\n\n    run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n    myrun = run.publish()\n    print(f\"kNN on {data.name}: {myrun.openml_url}\")\n</pre> # Easy benchmarking: for task_id in [115]:  # Add further tasks. Disclaimer: they might take some time     task = openml.tasks.get_task(task_id)     data = openml.datasets.get_dataset(task.dataset_id)     clf = neighbors.KNeighborsClassifier(n_neighbors=5)      run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)     myrun = run.publish()     print(f\"kNN on {data.name}: {myrun.openml_url}\") In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/flows_and_runs_tutorial/#flows-and-runs","title":"Flows and Runs\u00b6","text":"<p>This tutorial covers how to train/run a model and how to upload the results.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#train-machine-learning-models","title":"Train machine learning models\u00b6","text":"<p>Train a scikit-learn model on the data manually.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#runs-easily-explore-models","title":"Runs: Easily explore models\u00b6","text":"<p>We can run (many) scikit-learn algorithms on (many) OpenML tasks.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#it-also-works-with-pipelines","title":"It also works with pipelines\u00b6","text":"<p>When you need to handle 'dirty' data, build pipelines to model then automatically. To demonstrate this using the dataset <code>credit-a &lt;https://test.openml.org/d/16&gt;</code>_ via <code>task &lt;https://test.openml.org/t/96&gt;</code>_ as it contains both numerical and categorical variables and missing values in both.</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#running-flows-on-tasks-offline-for-later-upload","title":"Running flows on tasks offline for later upload\u00b6","text":"<p>For those scenarios where there is no access to internet, it is possible to run a model on a task without uploading results or flows to the server immediately.</p> <p>To perform the following line offline, it is required to have been called before such that the task is cached on the local openml cache directory:</p>"},{"location":"examples/30_extended/flows_and_runs_tutorial/#challenge","title":"Challenge\u00b6","text":"<p>Try to build the best possible models on several OpenML tasks, compare your results with the rest of the class and learn from them. Some tasks you could try (or browse openml.org):</p> <ul> <li>EEG eye state: data_id:<code>1471 &lt;https://www.openml.org/d/1471&gt;</code>, task_id:<code>14951 &lt;https://www.openml.org/t/14951&gt;</code></li> <li>Volcanoes on Venus: data_id:<code>1527 &lt;https://www.openml.org/d/1527&gt;</code>, task_id:<code>10103 &lt;https://www.openml.org/t/10103&gt;</code></li> <li>Walking activity: data_id:<code>1509 &lt;https://www.openml.org/d/1509&gt;</code>, task_id:<code>9945 &lt;https://www.openml.org/t/9945&gt;</code>, 150k instances.</li> <li>Covertype (Satellite): data_id:<code>150 &lt;https://www.openml.org/d/150&gt;</code>, task_id:<code>218 &lt;https://www.openml.org/t/218&gt;</code>, 500k instances.</li> <li>Higgs (Physics): data_id:<code>23512 &lt;https://www.openml.org/d/23512&gt;</code>, task_id:<code>52950 &lt;https://www.openml.org/t/52950&gt;</code>, 100k instances, missing values.</li> </ul>"},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/","title":"Plotting hyperparameter surfaces","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nimport numpy as np\n</pre> import openml import numpy as np In\u00a0[\u00a0]: Copied! <pre>df = openml.evaluations.list_evaluations_setups(\n    function=\"predictive_accuracy\",\n    flows=[8353],\n    tasks=[6],\n    output_format=\"dataframe\",\n    # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise,\n    # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary.\n    parameters_in_separate_columns=True,\n)\nprint(df.head(n=10))\n</pre> df = openml.evaluations.list_evaluations_setups(     function=\"predictive_accuracy\",     flows=[8353],     tasks=[6],     output_format=\"dataframe\",     # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise,     # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary.     parameters_in_separate_columns=True, ) print(df.head(n=10)) <p>We can see all the hyperparameter names in the columns of the dataframe:</p> In\u00a0[\u00a0]: Copied! <pre>for name in df.columns:\n    print(name)\n</pre> for name in df.columns:     print(name) <p>Next, we cast and transform the hyperparameters of interest (<code>C</code> and <code>gamma</code>) so that we can nicely plot them.</p> In\u00a0[\u00a0]: Copied! <pre>hyperparameters = [\"sklearn.svm.classes.SVC(16)_C\", \"sklearn.svm.classes.SVC(16)_gamma\"]\ndf[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10)\n</pre> hyperparameters = [\"sklearn.svm.classes.SVC(16)_C\", \"sklearn.svm.classes.SVC(16)_gamma\"] df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10) In\u00a0[\u00a0]: Copied! <pre>df.plot.hexbin(\n    x=\"sklearn.svm.classes.SVC(16)_C\",\n    y=\"sklearn.svm.classes.SVC(16)_gamma\",\n    C=\"value\",\n    reduce_C_function=np.mean,\n    gridsize=25,\n    title=\"SVM performance landscape\",\n)\n</pre> df.plot.hexbin(     x=\"sklearn.svm.classes.SVC(16)_C\",     y=\"sklearn.svm.classes.SVC(16)_gamma\",     C=\"value\",     reduce_C_function=np.mean,     gridsize=25,     title=\"SVM performance landscape\", ) In\u00a0[\u00a0]: Copied! <pre>import matplotlib.pyplot as plt\n\nfig, ax = plt.subplots()\n\nC = df[\"sklearn.svm.classes.SVC(16)_C\"]\ngamma = df[\"sklearn.svm.classes.SVC(16)_gamma\"]\nscore = df[\"value\"]\n\n# Plotting all evaluations:\nax.plot(C, gamma, \"ko\", ms=1)\n# Create a contour plot\ncntr = ax.tricontourf(C, gamma, score, levels=12, cmap=\"RdBu_r\")\n# Adjusting the colorbar\nfig.colorbar(cntr, ax=ax, label=\"accuracy\")\n# Adjusting the axis limits\nax.set(\n    xlim=(min(C), max(C)),\n    ylim=(min(gamma), max(gamma)),\n    xlabel=\"C (log10)\",\n    ylabel=\"gamma (log10)\",\n)\nax.set_title(\"SVM performance landscape\")\n# License: BSD 3-Clause\n</pre> import matplotlib.pyplot as plt  fig, ax = plt.subplots()  C = df[\"sklearn.svm.classes.SVC(16)_C\"] gamma = df[\"sklearn.svm.classes.SVC(16)_gamma\"] score = df[\"value\"]  # Plotting all evaluations: ax.plot(C, gamma, \"ko\", ms=1) # Create a contour plot cntr = ax.tricontourf(C, gamma, score, levels=12, cmap=\"RdBu_r\") # Adjusting the colorbar fig.colorbar(cntr, ax=ax, label=\"accuracy\") # Adjusting the axis limits ax.set(     xlim=(min(C), max(C)),     ylim=(min(gamma), max(gamma)),     xlabel=\"C (log10)\",     ylabel=\"gamma (log10)\", ) ax.set_title(\"SVM performance landscape\") # License: BSD 3-Clause"},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#plotting-hyperparameter-surfaces","title":"Plotting hyperparameter surfaces\u00b6","text":""},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#first-step-obtaining-the-data","title":"First step - obtaining the data\u00b6","text":"<p>First, we need to choose an SVM flow, for example 8353, and a task. Finding the IDs of them are not part of this tutorial, this could for example be done via the website.</p> <p>For this we use the function <code>list_evaluations_setup</code> which can automatically join evaluations conducted by the server with the hyperparameter settings extracted from the uploaded runs (called setup).</p>"},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#option-1-plotting-via-the-pandas-helper-functions","title":"Option 1 - plotting via the pandas helper functions\u00b6","text":""},{"location":"examples/30_extended/plot_svm_hyperparameters_tutorial/#option-2-plotting-via-matplotlib","title":"Option 2 - plotting via matplotlib\u00b6","text":""},{"location":"examples/30_extended/run_setup_tutorial/","title":"Run Setup","text":"In\u00a0[\u00a0]: Copied! <pre>import numpy as np\nimport openml\nfrom openml.extensions.sklearn import cat, cont\n\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder, FunctionTransformer\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.decomposition import TruncatedSVD\n</pre>  import numpy as np import openml from openml.extensions.sklearn import cat, cont  from sklearn.pipeline import make_pipeline, Pipeline from sklearn.compose import ColumnTransformer from sklearn.impute import SimpleImputer from sklearn.preprocessing import OneHotEncoder, FunctionTransformer from sklearn.ensemble import RandomForestClassifier from sklearn.decomposition import TruncatedSVD <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <ol> <li>Create a flow and use it to solve a task</li> </ol> <p>First, let's download the task that we are interested in</p> In\u00a0[\u00a0]: Copied! <pre>task = openml.tasks.get_task(6)\n</pre> task = openml.tasks.get_task(6) <p>we will create a fairly complex model, with many preprocessing components and many potential hyperparameters. Of course, the model can be as complex and as easy as you want it to be</p> In\u00a0[\u00a0]: Copied! <pre>cat_imp = make_pipeline(\n    OneHotEncoder(handle_unknown=\"ignore\", sparse=False),\n    TruncatedSVD(),\n)\ncont_imp = SimpleImputer(strategy=\"median\")\nct = ColumnTransformer([(\"cat\", cat_imp, cat), (\"cont\", cont_imp, cont)])\nmodel_original = Pipeline(\n    steps=[\n        (\"transform\", ct),\n        (\"estimator\", RandomForestClassifier()),\n    ]\n)\n</pre> cat_imp = make_pipeline(     OneHotEncoder(handle_unknown=\"ignore\", sparse=False),     TruncatedSVD(), ) cont_imp = SimpleImputer(strategy=\"median\") ct = ColumnTransformer([(\"cat\", cat_imp, cat), (\"cont\", cont_imp, cont)]) model_original = Pipeline(     steps=[         (\"transform\", ct),         (\"estimator\", RandomForestClassifier()),     ] ) <p>Let's change some hyperparameters. Of course, in any good application we would tune them using, e.g., Random Search or Bayesian Optimization, but for the purpose of this tutorial we set them to some specific values that might or might not be optimal</p> In\u00a0[\u00a0]: Copied! <pre>hyperparameters_original = {\n    \"estimator__criterion\": \"gini\",\n    \"estimator__n_estimators\": 50,\n    \"estimator__max_depth\": 10,\n    \"estimator__min_samples_leaf\": 1,\n}\nmodel_original.set_params(**hyperparameters_original)\n\n# solve the task and upload the result (this implicitly creates the flow)\nrun = openml.runs.run_model_on_task(model_original, task, avoid_duplicate_runs=False)\nrun_original = run.publish()  # this implicitly uploads the flow\n</pre> hyperparameters_original = {     \"estimator__criterion\": \"gini\",     \"estimator__n_estimators\": 50,     \"estimator__max_depth\": 10,     \"estimator__min_samples_leaf\": 1, } model_original.set_params(**hyperparameters_original)  # solve the task and upload the result (this implicitly creates the flow) run = openml.runs.run_model_on_task(model_original, task, avoid_duplicate_runs=False) run_original = run.publish()  # this implicitly uploads the flow In\u00a0[\u00a0]: Copied! <pre># obtain setup id (note that the setup id is assigned by the OpenML server -\n# therefore it was not yet available in our local copy of the run)\nrun_downloaded = openml.runs.get_run(run_original.run_id)\nsetup_id = run_downloaded.setup_id\n\n# after this, we can easily reinstantiate the model\nmodel_duplicate = openml.setups.initialize_model(setup_id)\n# it will automatically have all the hyperparameters set\n\n# and run the task again\nrun_duplicate = openml.runs.run_model_on_task(model_duplicate, task, avoid_duplicate_runs=False)\n</pre> # obtain setup id (note that the setup id is assigned by the OpenML server - # therefore it was not yet available in our local copy of the run) run_downloaded = openml.runs.get_run(run_original.run_id) setup_id = run_downloaded.setup_id  # after this, we can easily reinstantiate the model model_duplicate = openml.setups.initialize_model(setup_id) # it will automatically have all the hyperparameters set  # and run the task again run_duplicate = openml.runs.run_model_on_task(model_duplicate, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre># the run has stored all predictions in the field data content\nnp.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content)\n</pre> # the run has stored all predictions in the field data content np.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n\n# By: Jan N. van Rijn\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example()  # By: Jan N. van Rijn # License: BSD 3-Clause"},{"location":"examples/30_extended/run_setup_tutorial/#run-setup","title":"Run Setup\u00b6","text":"<p>One of the key features of the openml-python library is that is allows to reinstantiate flows with hyperparameter settings that were uploaded before. This tutorial uses the concept of setups. Although setups are not extensively described in the OpenML documentation (because most users will not directly use them), they form a important concept within OpenML distinguishing between hyperparameter configurations. A setup is the combination of a flow with all its hyperparameters set.</p> <p>A key requirement for reinstantiating a flow is to have the same scikit-learn version as the flow that was uploaded. However, this tutorial will upload the flow (that will later be reinstantiated) itself, so it can be ran with any scikit-learn version that is supported by this library. In this case, the requirement of the corresponding scikit-learn versions is automatically met.</p> <p>In this tutorial we will 1) Create a flow and use it to solve a task; 2) Download the flow, reinstantiate the model with same hyperparameters, and solve the same task again; 3) We will verify that the obtained results are exactly the same.</p>"},{"location":"examples/30_extended/run_setup_tutorial/#2-download-the-flow-and-solve-the-same-task-again","title":"2) Download the flow and solve the same task again.\u00b6","text":""},{"location":"examples/30_extended/run_setup_tutorial/#3-we-will-verify-that-the-obtained-results-are-exactly-the-same","title":"3) We will verify that the obtained results are exactly the same.\u00b6","text":""},{"location":"examples/30_extended/study_tutorial/","title":"Benchmark studies","text":"In\u00a0[\u00a0]: Copied! <pre>import uuid\n\nfrom sklearn.ensemble import RandomForestClassifier\n\nimport openml\n</pre> import uuid  from sklearn.ensemble import RandomForestClassifier  import openml In\u00a0[\u00a0]: Copied! <pre>studies = openml.study.list_studies(output_format=\"dataframe\", status=\"all\")\nprint(studies.head(n=10))\n</pre> studies = openml.study.list_studies(output_format=\"dataframe\", status=\"all\") print(studies.head(n=10)) <p>This is done based on the study ID.</p> In\u00a0[\u00a0]: Copied! <pre>study = openml.study.get_study(123)\nprint(study)\n</pre> study = openml.study.get_study(123) print(study) <p>Studies also features a description:</p> In\u00a0[\u00a0]: Copied! <pre>print(study.description)\n</pre> print(study.description) <p>Studies are a container for runs:</p> In\u00a0[\u00a0]: Copied! <pre>print(study.runs)\n</pre> print(study.runs) <p>And we can use the evaluation listing functionality to learn more about the evaluations available for the conducted runs:</p> In\u00a0[\u00a0]: Copied! <pre>evaluations = openml.evaluations.list_evaluations(\n    function=\"predictive_accuracy\",\n    output_format=\"dataframe\",\n    study=study.study_id,\n)\nprint(evaluations.head())\n</pre> evaluations = openml.evaluations.list_evaluations(     function=\"predictive_accuracy\",     output_format=\"dataframe\",     study=study.study_id, ) print(evaluations.head()) <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre># Model to be used\nclf = RandomForestClassifier()\n\n# We'll create a study with one run on 3 datasets present in the suite\ntasks = [115, 259, 307]\n\n# To verify\nsuite = openml.study.get_suite(1)\nprint(all([t_id in suite.tasks for t_id in tasks]))\n\nrun_ids = []\nfor task_id in tasks:\n    task = openml.tasks.get_task(task_id)\n    run = openml.runs.run_model_on_task(clf, task)\n    run.publish()\n    run_ids.append(run.run_id)\n\n# The study needs a machine-readable and unique alias. To obtain this,\n# we simply generate a random uuid.\nalias = uuid.uuid4().hex\n\nnew_study = openml.study.create_study(\n    name=\"Test-Study\",\n    description=\"Test study for the Python tutorial on studies\",\n    run_ids=run_ids,\n    alias=alias,\n    benchmark_suite=suite.study_id,\n)\nnew_study.publish()\nprint(new_study)\n</pre> # Model to be used clf = RandomForestClassifier()  # We'll create a study with one run on 3 datasets present in the suite tasks = [115, 259, 307]  # To verify suite = openml.study.get_suite(1) print(all([t_id in suite.tasks for t_id in tasks]))  run_ids = [] for task_id in tasks:     task = openml.tasks.get_task(task_id)     run = openml.runs.run_model_on_task(clf, task)     run.publish()     run_ids.append(run.run_id)  # The study needs a machine-readable and unique alias. To obtain this, # we simply generate a random uuid. alias = uuid.uuid4().hex  new_study = openml.study.create_study(     name=\"Test-Study\",     description=\"Test study for the Python tutorial on studies\",     run_ids=run_ids,     alias=alias,     benchmark_suite=suite.study_id, ) new_study.publish() print(new_study) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/study_tutorial/#benchmark-studies","title":"Benchmark studies\u00b6","text":"<p>How to list, download and upload benchmark studies. In contrast to benchmark suites which hold a list of tasks, studies hold a list of runs. As runs contain all information on flows and tasks, all required information about a study can be retrieved.</p>"},{"location":"examples/30_extended/study_tutorial/#listing-studies","title":"Listing studies\u00b6","text":"<ul> <li>Use the output_format parameter to select output type</li> <li>Default gives <code>dict</code>, but we'll use <code>dataframe</code> to obtain an easier-to-work-with data structure</li> </ul>"},{"location":"examples/30_extended/study_tutorial/#downloading-studies","title":"Downloading studies\u00b6","text":""},{"location":"examples/30_extended/study_tutorial/#uploading-studies","title":"Uploading studies\u00b6","text":"<p>Creating a study is as simple as creating any kind of other OpenML entity. In this examples we'll create a few runs for the OpenML-100 benchmark suite which is available on the OpenML test server.</p>"},{"location":"examples/30_extended/suites_tutorial/","title":"Benchmark suites","text":"In\u00a0[\u00a0]: Copied! <pre>import uuid\n\nimport numpy as np\n\nimport openml\n</pre> import uuid  import numpy as np  import openml In\u00a0[\u00a0]: Copied! <pre>suites = openml.study.list_suites(output_format=\"dataframe\", status=\"all\")\nprint(suites.head(n=10))\n</pre> suites = openml.study.list_suites(output_format=\"dataframe\", status=\"all\") print(suites.head(n=10)) <p>This is done based on the dataset ID.</p> In\u00a0[\u00a0]: Copied! <pre>suite = openml.study.get_suite(99)\nprint(suite)\n</pre> suite = openml.study.get_suite(99) print(suite) <p>Suites also feature a description:</p> In\u00a0[\u00a0]: Copied! <pre>print(suite.description)\n</pre> print(suite.description) <p>Suites are a container for tasks:</p> In\u00a0[\u00a0]: Copied! <pre>print(suite.tasks)\n</pre> print(suite.tasks) <p>And we can use the task listing functionality to learn more about them:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(output_format=\"dataframe\")\n</pre> tasks = openml.tasks.list_tasks(output_format=\"dataframe\") <p>Using <code>@</code> in pd.DataFrame.query accesses variables outside of the current dataframe.</p> In\u00a0[\u00a0]: Copied! <pre>tasks = tasks.query(\"tid in @suite.tasks\")\nprint(tasks.describe().transpose())\n</pre> tasks = tasks.query(\"tid in @suite.tasks\") print(tasks.describe().transpose()) <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>all_tasks = list(openml.tasks.list_tasks(output_format=\"dataframe\")[\"tid\"])\ntask_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20))\n\n# The study needs a machine-readable and unique alias. To obtain this,\n# we simply generate a random uuid.\n\nalias = uuid.uuid4().hex\n\nnew_suite = openml.study.create_benchmark_suite(\n    name=\"Test-Suite\",\n    description=\"Test suite for the Python tutorial on benchmark suites\",\n    task_ids=task_ids_for_suite,\n    alias=alias,\n)\nnew_suite.publish()\nprint(new_suite)\n</pre> all_tasks = list(openml.tasks.list_tasks(output_format=\"dataframe\")[\"tid\"]) task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20))  # The study needs a machine-readable and unique alias. To obtain this, # we simply generate a random uuid.  alias = uuid.uuid4().hex  new_suite = openml.study.create_benchmark_suite(     name=\"Test-Suite\",     description=\"Test suite for the Python tutorial on benchmark suites\",     task_ids=task_ids_for_suite,     alias=alias, ) new_suite.publish() print(new_suite) In\u00a0[\u00a0]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n# License: BSD 3-Clause\n</pre> openml.config.stop_using_configuration_for_example() # License: BSD 3-Clause"},{"location":"examples/30_extended/suites_tutorial/#benchmark-suites","title":"Benchmark suites\u00b6","text":"<p>How to list, download and upload benchmark suites.</p> <p>If you want to learn more about benchmark suites, check out our brief introductory tutorial \"Simple suites tutorial\" or the OpenML benchmark docs.</p>"},{"location":"examples/30_extended/suites_tutorial/#listing-suites","title":"Listing suites\u00b6","text":"<ul> <li>Use the output_format parameter to select output type</li> <li>Default gives <code>dict</code>, but we'll use <code>dataframe</code> to obtain an easier-to-work-with data structure</li> </ul>"},{"location":"examples/30_extended/suites_tutorial/#downloading-suites","title":"Downloading suites\u00b6","text":""},{"location":"examples/30_extended/suites_tutorial/#uploading-suites","title":"Uploading suites\u00b6","text":"<p>Uploading suites is as simple as uploading any kind of other OpenML entity - the only reason why we need so much code in this example is because we upload some random data.</p> <p>We'll take a random subset of at least ten tasks of all available tasks on the test server:</p>"},{"location":"examples/30_extended/task_manual_iteration_tutorial/","title":"Tasks: retrieving splits","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\n</pre> import openml <p>For this tutorial we will use the famous King+Rook versus King+Pawn on A7 dataset, which has the dataset ID 3 (dataset on OpenML), and for which there exist tasks with all important estimation procedures. It is small enough (less than 5000 samples) to efficiently use it in an example.</p> <p>We will first start with (task 233), which is a task with a holdout estimation procedure.</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 233\ntask = openml.tasks.get_task(task_id)\n</pre> task_id = 233 task = openml.tasks.get_task(task_id) <p>Now that we have a task object we can obtain the number of repetitions, folds and samples as defined by the task:</p> In\u00a0[\u00a0]: Copied! <pre>n_repeats, n_folds, n_samples = task.get_split_dimensions()\n</pre> n_repeats, n_folds, n_samples = task.get_split_dimensions() <ul> <li><code>n_repeats</code>: Number of times the model quality estimation is performed</li> <li><code>n_folds</code>: Number of folds per repeat</li> <li><code>n_samples</code>: How many data points to use. This is only relevant for learning curve tasks</li> </ul> <p>A list of all available estimation procedures is available here.</p> <p>Task <code>233</code> is a simple task using the holdout estimation procedure and therefore has only a single repeat, a single fold and a single sample size:</p> In\u00a0[\u00a0]: Copied! <pre>print(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>We can now retrieve the train/test split for this combination of repeats, folds and number of samples (indexing is zero-based). Usually, one would loop over all repeats, folds and sample sizes, but we can neglect this here as there is only a single repetition.</p> In\u00a0[\u00a0]: Copied! <pre>train_indices, test_indices = task.get_train_test_split_indices(\n    repeat=0,\n    fold=0,\n    sample=0,\n)\n\nprint(train_indices.shape, train_indices.dtype)\nprint(test_indices.shape, test_indices.dtype)\n</pre> train_indices, test_indices = task.get_train_test_split_indices(     repeat=0,     fold=0,     sample=0, )  print(train_indices.shape, train_indices.dtype) print(test_indices.shape, test_indices.dtype) <p>And then split the data based on this:</p> In\u00a0[\u00a0]: Copied! <pre>X, y = task.get_X_and_y(dataset_format=\"dataframe\")\nX_train = X.iloc[train_indices]\ny_train = y.iloc[train_indices]\nX_test = X.iloc[test_indices]\ny_test = y.iloc[test_indices]\n\nprint(\n    \"X_train.shape: {}, y_train.shape: {}, X_test.shape: {}, y_test.shape: {}\".format(\n        X_train.shape,\n        y_train.shape,\n        X_test.shape,\n        y_test.shape,\n    )\n)\n</pre> X, y = task.get_X_and_y(dataset_format=\"dataframe\") X_train = X.iloc[train_indices] y_train = y.iloc[train_indices] X_test = X.iloc[test_indices] y_test = y.iloc[test_indices]  print(     \"X_train.shape: {}, y_train.shape: {}, X_test.shape: {}, y_test.shape: {}\".format(         X_train.shape,         y_train.shape,         X_test.shape,         y_test.shape,     ) ) <p>Obviously, we can also retrieve cross-validation versions of the dataset used in task <code>233</code>:</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 3\ntask = openml.tasks.get_task(task_id)\nX, y = task.get_X_and_y(dataset_format=\"dataframe\")\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> task_id = 3 task = openml.tasks.get_task(task_id) X, y = task.get_X_and_y(dataset_format=\"dataframe\") n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>And then perform the aforementioned iteration over all splits:</p> In\u00a0[\u00a0]: Copied! <pre>for repeat_idx in range(n_repeats):\n    for fold_idx in range(n_folds):\n        for sample_idx in range(n_samples):\n            train_indices, test_indices = task.get_train_test_split_indices(\n                repeat=repeat_idx,\n                fold=fold_idx,\n                sample=sample_idx,\n            )\n            X_train = X.iloc[train_indices]\n            y_train = y.iloc[train_indices]\n            X_test = X.iloc[test_indices]\n            y_test = y.iloc[test_indices]\n\n            print(\n                \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"\n                \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(\n                    repeat_idx,\n                    fold_idx,\n                    sample_idx,\n                    X_train.shape,\n                    y_train.shape,\n                    X_test.shape,\n                    y_test.shape,\n                )\n            )\n</pre> for repeat_idx in range(n_repeats):     for fold_idx in range(n_folds):         for sample_idx in range(n_samples):             train_indices, test_indices = task.get_train_test_split_indices(                 repeat=repeat_idx,                 fold=fold_idx,                 sample=sample_idx,             )             X_train = X.iloc[train_indices]             y_train = y.iloc[train_indices]             X_test = X.iloc[test_indices]             y_test = y.iloc[test_indices]              print(                 \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"                 \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(                     repeat_idx,                     fold_idx,                     sample_idx,                     X_train.shape,                     y_train.shape,                     X_test.shape,                     y_test.shape,                 )             ) <p>And also versions with multiple repeats:</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 1767\ntask = openml.tasks.get_task(task_id)\nX, y = task.get_X_and_y(dataset_format=\"dataframe\")\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> task_id = 1767 task = openml.tasks.get_task(task_id) X, y = task.get_X_and_y(dataset_format=\"dataframe\") n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>And then again perform the aforementioned iteration over all splits:</p> In\u00a0[\u00a0]: Copied! <pre>for repeat_idx in range(n_repeats):\n    for fold_idx in range(n_folds):\n        for sample_idx in range(n_samples):\n            train_indices, test_indices = task.get_train_test_split_indices(\n                repeat=repeat_idx,\n                fold=fold_idx,\n                sample=sample_idx,\n            )\n            X_train = X.iloc[train_indices]\n            y_train = y.iloc[train_indices]\n            X_test = X.iloc[test_indices]\n            y_test = y.iloc[test_indices]\n\n            print(\n                \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"\n                \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(\n                    repeat_idx,\n                    fold_idx,\n                    sample_idx,\n                    X_train.shape,\n                    y_train.shape,\n                    X_test.shape,\n                    y_test.shape,\n                )\n            )\n</pre> for repeat_idx in range(n_repeats):     for fold_idx in range(n_folds):         for sample_idx in range(n_samples):             train_indices, test_indices = task.get_train_test_split_indices(                 repeat=repeat_idx,                 fold=fold_idx,                 sample=sample_idx,             )             X_train = X.iloc[train_indices]             y_train = y.iloc[train_indices]             X_test = X.iloc[test_indices]             y_test = y.iloc[test_indices]              print(                 \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"                 \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(                     repeat_idx,                     fold_idx,                     sample_idx,                     X_train.shape,                     y_train.shape,                     X_test.shape,                     y_test.shape,                 )             ) <p>And finally a task based on learning curves:</p> In\u00a0[\u00a0]: Copied! <pre>task_id = 1702\ntask = openml.tasks.get_task(task_id)\nX, y = task.get_X_and_y(dataset_format=\"dataframe\")\nn_repeats, n_folds, n_samples = task.get_split_dimensions()\nprint(\n    \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(\n        task_id,\n        n_repeats,\n        n_folds,\n        n_samples,\n    )\n)\n</pre> task_id = 1702 task = openml.tasks.get_task(task_id) X, y = task.get_X_and_y(dataset_format=\"dataframe\") n_repeats, n_folds, n_samples = task.get_split_dimensions() print(     \"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.\".format(         task_id,         n_repeats,         n_folds,         n_samples,     ) ) <p>And then again perform the aforementioned iteration over all splits:</p> In\u00a0[\u00a0]: Copied! <pre>for repeat_idx in range(n_repeats):\n    for fold_idx in range(n_folds):\n        for sample_idx in range(n_samples):\n            train_indices, test_indices = task.get_train_test_split_indices(\n                repeat=repeat_idx,\n                fold=fold_idx,\n                sample=sample_idx,\n            )\n            X_train = X.iloc[train_indices]\n            y_train = y.iloc[train_indices]\n            X_test = X.iloc[test_indices]\n            y_test = y.iloc[test_indices]\n\n            print(\n                \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"\n                \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(\n                    repeat_idx,\n                    fold_idx,\n                    sample_idx,\n                    X_train.shape,\n                    y_train.shape,\n                    X_test.shape,\n                    y_test.shape,\n                )\n            )\n# License: BSD 3-Clause\n</pre> for repeat_idx in range(n_repeats):     for fold_idx in range(n_folds):         for sample_idx in range(n_samples):             train_indices, test_indices = task.get_train_test_split_indices(                 repeat=repeat_idx,                 fold=fold_idx,                 sample=sample_idx,             )             X_train = X.iloc[train_indices]             y_train = y.iloc[train_indices]             X_test = X.iloc[test_indices]             y_test = y.iloc[test_indices]              print(                 \"Repeat #{}, fold #{}, samples {}: X_train.shape: {}, \"                 \"y_train.shape {}, X_test.shape {}, y_test.shape {}\".format(                     repeat_idx,                     fold_idx,                     sample_idx,                     X_train.shape,                     y_train.shape,                     X_test.shape,                     y_test.shape,                 )             ) # License: BSD 3-Clause"},{"location":"examples/30_extended/task_manual_iteration_tutorial/#tasks-retrieving-splits","title":"Tasks: retrieving splits\u00b6","text":"<p>Tasks define a target and a train/test split. Normally, they are the input to the function <code>openml.runs.run_model_on_task</code> which automatically runs the model on all splits of the task. However, sometimes it is necessary to manually split a dataset to perform experiments outside of the functions provided by OpenML. One such example is in the benchmark library HPOBench which extensively uses data from OpenML, but not OpenML's functionality to conduct runs.</p>"},{"location":"examples/30_extended/tasks_tutorial/","title":"Tasks","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nfrom openml.tasks import TaskType\nimport pandas as pd\n</pre> import openml from openml.tasks import TaskType import pandas as pd <p>Tasks are identified by IDs and can be accessed in two different ways:</p> <ol> <li>In a list providing basic information on all tasks available on OpenML. This function will not download the actual tasks, but will instead download meta data that can be used to filter the tasks and retrieve a set of IDs. We can filter this list, for example, we can only list tasks having a special tag or only tasks for a specific target such as supervised classification.</li> <li>A single task by its ID. It contains all meta information, the target metric, the splits and an iterator which can be used to access the splits in a useful manner.</li> </ol> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(\n    task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format=\"dataframe\"\n)\nprint(tasks.columns)\nprint(f\"First 5 of {len(tasks)} tasks:\")\nprint(tasks.head())\n</pre> tasks = openml.tasks.list_tasks(     task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format=\"dataframe\" ) print(tasks.columns) print(f\"First 5 of {len(tasks)} tasks:\") print(tasks.head()) <p>We can filter the list of tasks to only contain datasets with more than 500 samples, but less than 1000 samples:</p> In\u00a0[\u00a0]: Copied! <pre>filtered_tasks = tasks.query(\"NumberOfInstances &gt; 500 and NumberOfInstances &lt; 1000\")\nprint(list(filtered_tasks.index))\n</pre> filtered_tasks = tasks.query(\"NumberOfInstances &gt; 500 and NumberOfInstances &lt; 1000\") print(list(filtered_tasks.index)) In\u00a0[\u00a0]: Copied! <pre># Number of tasks\nprint(len(filtered_tasks))\n</pre> # Number of tasks print(len(filtered_tasks)) <p>Then, we can further restrict the tasks to all have the same resampling strategy:</p> In\u00a0[\u00a0]: Copied! <pre>filtered_tasks = filtered_tasks.query('estimation_procedure == \"10-fold Crossvalidation\"')\nprint(list(filtered_tasks.index))\n</pre> filtered_tasks = filtered_tasks.query('estimation_procedure == \"10-fold Crossvalidation\"') print(list(filtered_tasks.index)) In\u00a0[\u00a0]: Copied! <pre># Number of tasks\nprint(len(filtered_tasks))\n</pre> # Number of tasks print(len(filtered_tasks)) <p>Resampling strategies can be found on the OpenML Website.</p> <p>Similar to listing tasks by task type, we can list tasks by tags:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(tag=\"OpenML100\", output_format=\"dataframe\")\nprint(f\"First 5 of {len(tasks)} tasks:\")\nprint(tasks.head())\n</pre> tasks = openml.tasks.list_tasks(tag=\"OpenML100\", output_format=\"dataframe\") print(f\"First 5 of {len(tasks)} tasks:\") print(tasks.head()) <p>Furthermore, we can list tasks based on the dataset id:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(data_id=1471, output_format=\"dataframe\")\nprint(f\"First 5 of {len(tasks)} tasks:\")\nprint(tasks.head())\n</pre> tasks = openml.tasks.list_tasks(data_id=1471, output_format=\"dataframe\") print(f\"First 5 of {len(tasks)} tasks:\") print(tasks.head()) <p>In addition, a size limit and an offset can be applied both separately and simultaneously:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(size=10, offset=50, output_format=\"dataframe\")\nprint(tasks)\n</pre> tasks = openml.tasks.list_tasks(size=10, offset=50, output_format=\"dataframe\") print(tasks) <p>OpenML 100 is a curated list of 100 tasks to start using OpenML. They are all supervised classification tasks with more than 500 instances and less than 50000 instances per task. To make things easier, the tasks do not contain highly unbalanced data and sparse data. However, the tasks include missing values and categorical features. You can find out more about the OpenML 100 on the OpenML benchmarking page.</p> <p>Finally, it is also possible to list all tasks on OpenML with:</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(output_format=\"dataframe\")\nprint(len(tasks))\n</pre> tasks = openml.tasks.list_tasks(output_format=\"dataframe\") print(len(tasks)) In\u00a0[\u00a0]: Copied! <pre>tasks.query('name==\"eeg-eye-state\"')\n</pre> tasks.query('name==\"eeg-eye-state\"') In\u00a0[\u00a0]: Copied! <pre>task_id = 31\ntask = openml.tasks.get_task(task_id)\n</pre> task_id = 31 task = openml.tasks.get_task(task_id) In\u00a0[\u00a0]: Copied! <pre># Properties of the task are stored as member variables:\nprint(task)\n</pre> # Properties of the task are stored as member variables: print(task) In\u00a0[\u00a0]: Copied! <pre># And:\n\nids = [2, 1891, 31, 9983]\ntasks = openml.tasks.get_tasks(ids)\nprint(tasks[0])\n</pre> # And:  ids = [2, 1891, 31, 9983] tasks = openml.tasks.get_tasks(ids) print(tasks[0]) <p>We'll use the test server for the rest of this tutorial.</p> <p>.. warning:: .. include:: ../../test_server_usage_warning.txt</p> In\u00a0[\u00a0]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() In\u00a0[\u00a0]: Copied! <pre>try:\n    my_task = openml.tasks.create_task(\n        task_type=TaskType.SUPERVISED_CLASSIFICATION,\n        dataset_id=128,\n        target_name=\"class\",\n        evaluation_measure=\"predictive_accuracy\",\n        estimation_procedure_id=1,\n    )\n    my_task.publish()\nexcept openml.exceptions.OpenMLServerException as e:\n    # Error code for 'task already exists'\n    if e.code == 614:\n        # Lookup task\n        tasks = openml.tasks.list_tasks(data_id=128, output_format=\"dataframe\")\n        tasks = tasks.query(\n            'task_type == \"Supervised Classification\" '\n            'and estimation_procedure == \"10-fold Crossvalidation\" '\n            'and evaluation_measures == \"predictive_accuracy\"'\n        )\n        task_id = tasks.loc[:, \"tid\"].values[0]\n        print(\"Task already exists. Task ID is\", task_id)\n</pre> try:     my_task = openml.tasks.create_task(         task_type=TaskType.SUPERVISED_CLASSIFICATION,         dataset_id=128,         target_name=\"class\",         evaluation_measure=\"predictive_accuracy\",         estimation_procedure_id=1,     )     my_task.publish() except openml.exceptions.OpenMLServerException as e:     # Error code for 'task already exists'     if e.code == 614:         # Lookup task         tasks = openml.tasks.list_tasks(data_id=128, output_format=\"dataframe\")         tasks = tasks.query(             'task_type == \"Supervised Classification\" '             'and estimation_procedure == \"10-fold Crossvalidation\" '             'and evaluation_measures == \"predictive_accuracy\"'         )         task_id = tasks.loc[:, \"tid\"].values[0]         print(\"Task already exists. Task ID is\", task_id) In\u00a0[\u00a0]: Copied! <pre># reverting to prod server\nopenml.config.stop_using_configuration_for_example()\n</pre> # reverting to prod server openml.config.stop_using_configuration_for_example() <ul> <li>Complete list of task types.</li> <li>Complete list of model estimation procedures.</li> <li>Complete list of evaluation measures.</li> </ul> <p>License: BSD 3-Clause</p>"},{"location":"examples/30_extended/tasks_tutorial/#tasks","title":"Tasks\u00b6","text":"<p>A tutorial on how to list and download tasks.</p>"},{"location":"examples/30_extended/tasks_tutorial/#listing-tasks","title":"Listing tasks\u00b6","text":"<p>We will start by simply listing only supervised classification tasks.</p> <p>openml.tasks.list_tasks() returns a dictionary of dictionaries by default, but we request a pandas dataframe instead to have better visualization capabilities and easier access:</p>"},{"location":"examples/30_extended/tasks_tutorial/#exercise","title":"Exercise\u00b6","text":"<p>Search for the tasks on the 'eeg-eye-state' dataset.</p>"},{"location":"examples/30_extended/tasks_tutorial/#downloading-tasks","title":"Downloading tasks\u00b6","text":"<p>We provide two functions to download tasks, one which downloads only a single task by its ID, and one which takes a list of IDs and downloads all of these tasks:</p>"},{"location":"examples/30_extended/tasks_tutorial/#creating-tasks","title":"Creating tasks\u00b6","text":"<p>You can also create new tasks. Take the following into account:</p> <ul> <li>You can only create tasks on active datasets</li> <li>For now, only the following tasks are supported: classification, regression, clustering, and learning curve analysis.</li> <li>For now, tasks can only be created on a single dataset.</li> <li>The exact same task must not already exist.</li> </ul> <p>Creating a task requires the following input:</p> <ul> <li>task_type: The task type ID, required (see below). Required.</li> <li>dataset_id: The dataset ID. Required.</li> <li>target_name: The name of the attribute you aim to predict. Optional.</li> <li>estimation_procedure_id : The ID of the estimation procedure used to create train-test splits. Optional.</li> <li>evaluation_measure: The name of the evaluation measure. Optional.</li> <li>Any additional inputs for specific tasks</li> </ul> <p>It is best to leave the evaluation measure open if there is no strong prerequisite for a specific measure. OpenML will always compute all appropriate measures and you can filter or sort results on your favourite measure afterwards. Only add an evaluation measure if necessary (e.g. when other measure make no sense), since it will create a new task, which scatters results across tasks.</p>"},{"location":"examples/30_extended/tasks_tutorial/#example","title":"Example\u00b6","text":"<p>Let's create a classification task on a dataset. In this example we will do this on the Iris dataset (ID=128 (on test server)). We'll use 10-fold cross-validation (ID=1), and predictive accuracy as the predefined measure (this can also be left open). If a task with these parameters exists, we will get an appropriate exception. If such a task doesn't exist, a task will be created and the corresponding task_id will be returned.</p>"},{"location":"examples/40_paper/2015_neurips_feurer_example/","title":"Feurer et al. (2015)","text":"In\u00a0[\u00a0]: Copied! <pre>import pandas as pd\n\nimport openml\n</pre> import pandas as pd  import openml <p>List of dataset IDs given in the supplementary material of Feurer et al.: https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip</p> In\u00a0[\u00a0]: Copied! <pre>dataset_ids = [\n    3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,\n    57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,\n    390, 391, 392, 393, 395, 396, 398, 399, 401, 554, 679, 715, 718, 720, 722,\n    723, 727, 728, 734, 735, 737, 740, 741, 743, 751, 752, 761, 772, 797, 799,\n    803, 806, 807, 813, 816, 819, 821, 822, 823, 833, 837, 843, 845, 846, 847,\n    849, 866, 871, 881, 897, 901, 903, 904, 910, 912, 913, 914, 917, 923, 930,\n    934, 953, 958, 959, 962, 966, 971, 976, 977, 978, 979, 980, 991, 993, 995,\n    1000, 1002, 1018, 1019, 1020, 1021, 1036, 1040, 1041, 1049, 1050, 1053,\n    1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,\n    1134, 1138, 1139, 1142, 1146, 1161, 1166,\n]\n</pre> dataset_ids = [     3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,     57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,     390, 391, 392, 393, 395, 396, 398, 399, 401, 554, 679, 715, 718, 720, 722,     723, 727, 728, 734, 735, 737, 740, 741, 743, 751, 752, 761, 772, 797, 799,     803, 806, 807, 813, 816, 819, 821, 822, 823, 833, 837, 843, 845, 846, 847,     849, 866, 871, 881, 897, 901, 903, 904, 910, 912, 913, 914, 917, 923, 930,     934, 953, 958, 959, 962, 966, 971, 976, 977, 978, 979, 980, 991, 993, 995,     1000, 1002, 1018, 1019, 1020, 1021, 1036, 1040, 1041, 1049, 1050, 1053,     1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,     1134, 1138, 1139, 1142, 1146, 1161, 1166, ] <p>The dataset IDs could be used directly to load the dataset and split the data into a training set and a test set. However, to be reproducible, we will first obtain the respective tasks from OpenML, which define both the target feature and the train/test split.</p> <p>.. note:: It is discouraged to work directly on datasets and only provide dataset IDs in a paper as this does not allow reproducibility (unclear splitting). Please do not use datasets but the respective tasks as basis for a paper and publish task IDS. This example is only given to showcase the use of OpenML-Python for a published paper and as a warning on how not to do it. Please check the <code>OpenML documentation of tasks &lt;https://docs.openml.org/#tasks&gt;</code>_ if you want to learn more about them.</p> <p>This lists both active and inactive tasks (because of <code>status='all'</code>). Unfortunately, this is necessary as some of the datasets contain issues found after the publication and became deactivated, which also deactivated the tasks on them. More information on active or inactive datasets can be found in the online docs.</p> In\u00a0[\u00a0]: Copied! <pre>tasks = openml.tasks.list_tasks(\n    task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION,\n    status=\"all\",\n    output_format=\"dataframe\",\n)\n\n# Query only those with holdout as the resampling startegy.\ntasks = tasks.query('estimation_procedure == \"33% Holdout set\"')\n\ntask_ids = []\nfor did in dataset_ids:\n    tasks_ = list(tasks.query(\"did == {}\".format(did)).tid)\n    if len(tasks_) &gt;= 1:  # if there are multiple task, take the one with lowest ID (oldest).\n        task_id = min(tasks_)\n    else:\n        raise ValueError(did)\n\n    # Optional - Check that the task has the same target attribute as the\n    # dataset default target attribute\n    # (disabled for this example as it needs to run fast to be rendered online)\n    # task = openml.tasks.get_task(task_id)\n    # dataset = task.get_dataset()\n    # if task.target_name != dataset.default_target_attribute:\n    #     raise ValueError(\n    #         (task.target_name, dataset.default_target_attribute)\n    #     )\n\n    task_ids.append(task_id)\n\nassert len(task_ids) == 140\ntask_ids.sort()\n\n# These are the tasks to work with:\nprint(task_ids)\n\n# License: BSD 3-Clause\n</pre> tasks = openml.tasks.list_tasks(     task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION,     status=\"all\",     output_format=\"dataframe\", )  # Query only those with holdout as the resampling startegy. tasks = tasks.query('estimation_procedure == \"33% Holdout set\"')  task_ids = [] for did in dataset_ids:     tasks_ = list(tasks.query(\"did == {}\".format(did)).tid)     if len(tasks_) &gt;= 1:  # if there are multiple task, take the one with lowest ID (oldest).         task_id = min(tasks_)     else:         raise ValueError(did)      # Optional - Check that the task has the same target attribute as the     # dataset default target attribute     # (disabled for this example as it needs to run fast to be rendered online)     # task = openml.tasks.get_task(task_id)     # dataset = task.get_dataset()     # if task.target_name != dataset.default_target_attribute:     #     raise ValueError(     #         (task.target_name, dataset.default_target_attribute)     #     )      task_ids.append(task_id)  assert len(task_ids) == 140 task_ids.sort()  # These are the tasks to work with: print(task_ids)  # License: BSD 3-Clause"},{"location":"examples/40_paper/2015_neurips_feurer_example/#feurer-et-al-2015","title":"Feurer et al. (2015)\u00b6","text":"<p>A tutorial on how to get the datasets used in the paper introducing Auto-sklearn by Feurer et al..</p> <p>Auto-sklearn website: https://automl.github.io/auto-sklearn/</p>"},{"location":"examples/40_paper/2015_neurips_feurer_example/#publication","title":"Publication\u00b6","text":"<p>| Efficient and Robust Automated Machine Learning | Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter | In Advances in Neural Information Processing Systems 28, 2015 | Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf</p>"},{"location":"examples/40_paper/2018_ida_strang_example/","title":"Strang et al. (2018)","text":"In\u00a0[\u00a0]: Copied! <pre>import matplotlib.pyplot as plt\nimport openml\nimport pandas as pd\n</pre> import matplotlib.pyplot as plt import openml import pandas as pd <p>A basic step for each data-mining or machine learning task is to determine which model to choose based on the problem and the data at hand. In this work we investigate when non-linear classifiers outperform linear classifiers by means of a large scale experiment.</p> <p>The paper is accompanied with a study object, containing all relevant tasks and runs (<code>study_id=123</code>). The paper features three experiment classes: Support Vector Machines (SVM), Neural Networks (NN) and Decision Trees (DT). This example demonstrates how to reproduce the plots, comparing two classifiers given the OpenML flow ids. Note that this allows us to reproduce the SVM and NN experiment, but not the DT experiment, as this requires a bit more effort to distinguish the same flow with different hyperparameter values.</p> In\u00a0[\u00a0]: Copied! <pre>study_id = 123\n# for comparing svms: flow_ids = [7754, 7756]\n# for comparing nns: flow_ids = [7722, 7729]\n# for comparing dts: flow_ids = [7725], differentiate on hyper-parameter value\nclassifier_family = \"SVM\"\nflow_ids = [7754, 7756]\nmeasure = \"predictive_accuracy\"\nmeta_features = [\"NumberOfInstances\", \"NumberOfFeatures\"]\nclass_values = [\"non-linear better\", \"linear better\", \"equal\"]\n\n# Downloads all evaluation records related to this study\nevaluations = openml.evaluations.list_evaluations(\n    measure, size=None, flows=flow_ids, study=study_id, output_format=\"dataframe\"\n)\n# gives us a table with columns data_id, flow1_value, flow2_value\nevaluations = evaluations.pivot(index=\"data_id\", columns=\"flow_id\", values=\"value\").dropna()\n# downloads all data qualities (for scatter plot)\ndata_qualities = openml.datasets.list_datasets(\n    data_id=list(evaluations.index.values), output_format=\"dataframe\"\n)\n# removes irrelevant data qualities\ndata_qualities = data_qualities[meta_features]\n# makes a join between evaluation table and data qualities table,\n# now we have columns data_id, flow1_value, flow2_value, meta_feature_1,\n# meta_feature_2\nevaluations = evaluations.join(data_qualities, how=\"inner\")\n\n# adds column that indicates the difference between the two classifiers\nevaluations[\"diff\"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]]\n</pre> study_id = 123 # for comparing svms: flow_ids = [7754, 7756] # for comparing nns: flow_ids = [7722, 7729] # for comparing dts: flow_ids = [7725], differentiate on hyper-parameter value classifier_family = \"SVM\" flow_ids = [7754, 7756] measure = \"predictive_accuracy\" meta_features = [\"NumberOfInstances\", \"NumberOfFeatures\"] class_values = [\"non-linear better\", \"linear better\", \"equal\"]  # Downloads all evaluation records related to this study evaluations = openml.evaluations.list_evaluations(     measure, size=None, flows=flow_ids, study=study_id, output_format=\"dataframe\" ) # gives us a table with columns data_id, flow1_value, flow2_value evaluations = evaluations.pivot(index=\"data_id\", columns=\"flow_id\", values=\"value\").dropna() # downloads all data qualities (for scatter plot) data_qualities = openml.datasets.list_datasets(     data_id=list(evaluations.index.values), output_format=\"dataframe\" ) # removes irrelevant data qualities data_qualities = data_qualities[meta_features] # makes a join between evaluation table and data qualities table, # now we have columns data_id, flow1_value, flow2_value, meta_feature_1, # meta_feature_2 evaluations = evaluations.join(data_qualities, how=\"inner\")  # adds column that indicates the difference between the two classifiers evaluations[\"diff\"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]] <p>makes the s-plot</p> In\u00a0[\u00a0]: Copied! <pre>fig_splot, ax_splot = plt.subplots()\nax_splot.plot(range(len(evaluations)), sorted(evaluations[\"diff\"]))\nax_splot.set_title(classifier_family)\nax_splot.set_xlabel(\"Dataset (sorted)\")\nax_splot.set_ylabel(\"difference between linear and non-linear classifier\")\nax_splot.grid(linestyle=\"--\", axis=\"y\")\nplt.show()\n</pre> fig_splot, ax_splot = plt.subplots() ax_splot.plot(range(len(evaluations)), sorted(evaluations[\"diff\"])) ax_splot.set_title(classifier_family) ax_splot.set_xlabel(\"Dataset (sorted)\") ax_splot.set_ylabel(\"difference between linear and non-linear classifier\") ax_splot.grid(linestyle=\"--\", axis=\"y\") plt.show() <p>adds column that indicates the difference between the two classifiers, needed for the scatter plot</p> In\u00a0[\u00a0]: Copied! <pre>def determine_class(val_lin, val_nonlin):\n    if val_lin &lt; val_nonlin:\n        return class_values[0]\n    elif val_nonlin &lt; val_lin:\n        return class_values[1]\n    else:\n        return class_values[2]\n\n\nevaluations[\"class\"] = evaluations.apply(\n    lambda row: determine_class(row[flow_ids[0]], row[flow_ids[1]]), axis=1\n)\n\n# does the plotting and formatting\nfig_scatter, ax_scatter = plt.subplots()\nfor class_val in class_values:\n    df_class = evaluations[evaluations[\"class\"] == class_val]\n    plt.scatter(df_class[meta_features[0]], df_class[meta_features[1]], label=class_val)\nax_scatter.set_title(classifier_family)\nax_scatter.set_xlabel(meta_features[0])\nax_scatter.set_ylabel(meta_features[1])\nax_scatter.legend()\nax_scatter.set_xscale(\"log\")\nax_scatter.set_yscale(\"log\")\nplt.show()\n</pre> def determine_class(val_lin, val_nonlin):     if val_lin &lt; val_nonlin:         return class_values[0]     elif val_nonlin &lt; val_lin:         return class_values[1]     else:         return class_values[2]   evaluations[\"class\"] = evaluations.apply(     lambda row: determine_class(row[flow_ids[0]], row[flow_ids[1]]), axis=1 )  # does the plotting and formatting fig_scatter, ax_scatter = plt.subplots() for class_val in class_values:     df_class = evaluations[evaluations[\"class\"] == class_val]     plt.scatter(df_class[meta_features[0]], df_class[meta_features[1]], label=class_val) ax_scatter.set_title(classifier_family) ax_scatter.set_xlabel(meta_features[0]) ax_scatter.set_ylabel(meta_features[1]) ax_scatter.legend() ax_scatter.set_xscale(\"log\") ax_scatter.set_yscale(\"log\") plt.show() <p>makes a scatter plot where each data point represents the performance of the two algorithms on various axis (not in the paper)</p> In\u00a0[\u00a0]: Copied! <pre>fig_diagplot, ax_diagplot = plt.subplots()\nax_diagplot.grid(linestyle=\"--\")\nax_diagplot.plot([0, 1], ls=\"-\", color=\"black\")\nax_diagplot.plot([0.2, 1.2], ls=\"--\", color=\"black\")\nax_diagplot.plot([-0.2, 0.8], ls=\"--\", color=\"black\")\nax_diagplot.scatter(evaluations[flow_ids[0]], evaluations[flow_ids[1]])\nax_diagplot.set_xlabel(measure)\nax_diagplot.set_ylabel(measure)\nplt.show()\n# License: BSD 3-Clause\n</pre> fig_diagplot, ax_diagplot = plt.subplots() ax_diagplot.grid(linestyle=\"--\") ax_diagplot.plot([0, 1], ls=\"-\", color=\"black\") ax_diagplot.plot([0.2, 1.2], ls=\"--\", color=\"black\") ax_diagplot.plot([-0.2, 0.8], ls=\"--\", color=\"black\") ax_diagplot.scatter(evaluations[flow_ids[0]], evaluations[flow_ids[1]]) ax_diagplot.set_xlabel(measure) ax_diagplot.set_ylabel(measure) plt.show() # License: BSD 3-Clause"},{"location":"examples/40_paper/2018_ida_strang_example/#strang-et-al-2018","title":"Strang et al. (2018)\u00b6","text":"<p>A tutorial on how to reproduce the analysis conducted for Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML.</p>"},{"location":"examples/40_paper/2018_ida_strang_example/#publication","title":"Publication\u00b6","text":"<p>| Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML | Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank Hutter | In Advances in Intelligent Data Analysis XVII 17th International Symposium, 2018 | Available at https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25</p>"},{"location":"examples/40_paper/2018_kdd_rijn_example/","title":"van Rijn and Hutter (2018)","text":"<p>With the advent of automated machine learning, automated hyperparameter optimization methods are by now routinely used in data mining. However, this progress is not yet matched by equal progress on automatic analyses that yield information beyond performance-optimizing hyperparameter settings. In this example, we aim to answer the following two questions: Given an algorithm, what are generally its most important hyperparameters?</p> <p>This work is carried out on the OpenML-100 benchmark suite, which can be obtained by <code>openml.study.get_suite('OpenML100')</code>. In this example, we conduct the experiment on the Support Vector Machine (<code>flow_id=7707</code>) with specific kernel (we will perform a post-process filter operation for this). We should set some other experimental parameters (number of results per task, evaluation measure and the number of trees of the internal functional Anova) before the fun can begin.</p> <p>Note that we simplify the example in several ways:</p> <ol> <li>We only consider numerical hyperparameters</li> <li>We consider all hyperparameters that are numerical (in reality, some hyperparameters might be inactive (e.g., <code>degree</code>) or irrelevant (e.g., <code>random_state</code>)</li> <li>We assume all hyperparameters to be on uniform scale</li> </ol> <p>Any difference in conclusion between the actual paper and the presented results is most likely due to one of these simplifications. For example, the hyperparameter C looks rather insignificant, whereas it is quite important when it is put on a log-scale. All these simplifications can be addressed by defining a ConfigSpace. For a more elaborated example that uses this, please see: https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401</p> In\u00a0[\u00a0]: Copied! <pre>suite = openml.study.get_suite(\"OpenML100\")\nflow_id = 7707\nparameter_filters = {\"sklearn.svm.classes.SVC(17)_kernel\": \"sigmoid\"}\nevaluation_measure = \"predictive_accuracy\"\nlimit_per_task = 500\nlimit_nr_tasks = 15\nn_trees = 16\n\nfanova_results = []\n# we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the\n# communication with OpenML, and for iterated experimenting it is better to cache the results in a local file.\nfor idx, task_id in enumerate(suite.tasks):\n    if limit_nr_tasks is not None and idx &gt;= limit_nr_tasks:\n        continue\n    print(\n        \"Starting with task %d (%d/%d)\"\n        % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks)\n    )\n    # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop)\n    evals = openml.evaluations.list_evaluations_setups(\n        evaluation_measure,\n        flows=[flow_id],\n        tasks=[task_id],\n        size=limit_per_task,\n        output_format=\"dataframe\",\n    )\n\n    performance_column = \"value\"\n    # make a DataFrame consisting of all hyperparameters (which is a dict in setup['parameters']) and the performance\n    # value (in setup['value']). The following line looks a bit complicated, but combines 2 tasks: a) combine\n    # hyperparameters and performance data in a single dict, b) cast hyperparameter values to the appropriate format\n    # Note that the ``json.loads(...)`` requires the content to be in JSON format, which is only the case for\n    # scikit-learn setups (and even there some legacy setups might violate this requirement). It will work for the\n    # setups that belong to the flows embedded in this example though.\n    try:\n        setups_evals = pd.DataFrame(\n            [\n                dict(\n                    **{name: json.loads(value) for name, value in setup[\"parameters\"].items()},\n                    **{performance_column: setup[performance_column]}\n                )\n                for _, setup in evals.iterrows()\n            ]\n        )\n    except json.decoder.JSONDecodeError as e:\n        print(\"Task %d error: %s\" % (task_id, e))\n        continue\n    # apply our filters, to have only the setups that comply to the hyperparameters we want\n    for filter_key, filter_value in parameter_filters.items():\n        setups_evals = setups_evals[setups_evals[filter_key] == filter_value]\n    # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters,\n    # the fanova library needs to be informed by using a configspace object.\n    setups_evals = setups_evals.select_dtypes(include=[\"int64\", \"float64\"])\n    # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``,\n    # ``verbose``.\n    setups_evals = setups_evals[\n        [\n            c\n            for c in list(setups_evals)\n            if len(setups_evals[c].unique()) &gt; 1 or c == performance_column\n        ]\n    ]\n    # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g.,\n    # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out:\n\n    # determine x values to pass to fanova library\n    parameter_names = [\n        pname for pname in setups_evals.columns.to_numpy() if pname != performance_column\n    ]\n    evaluator = fanova.fanova.fANOVA(\n        X=setups_evals[parameter_names].to_numpy(),\n        Y=setups_evals[performance_column].to_numpy(),\n        n_trees=n_trees,\n    )\n    for idx, pname in enumerate(parameter_names):\n        try:\n            fanova_results.append(\n                {\n                    \"hyperparameter\": pname.split(\".\")[-1],\n                    \"fanova\": evaluator.quantify_importance([idx])[(idx,)][\"individual importance\"],\n                }\n            )\n        except RuntimeError as e:\n            # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant\n            # for all configurations (there is no variance). We will skip these tasks (like the authors did in the\n            # paper).\n            print(\"Task %d error: %s\" % (task_id, e))\n            continue\n\n# transform ``fanova_results`` from a list of dicts into a DataFrame\nfanova_results = pd.DataFrame(fanova_results)\n</pre> suite = openml.study.get_suite(\"OpenML100\") flow_id = 7707 parameter_filters = {\"sklearn.svm.classes.SVC(17)_kernel\": \"sigmoid\"} evaluation_measure = \"predictive_accuracy\" limit_per_task = 500 limit_nr_tasks = 15 n_trees = 16  fanova_results = [] # we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the # communication with OpenML, and for iterated experimenting it is better to cache the results in a local file. for idx, task_id in enumerate(suite.tasks):     if limit_nr_tasks is not None and idx &gt;= limit_nr_tasks:         continue     print(         \"Starting with task %d (%d/%d)\"         % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks)     )     # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop)     evals = openml.evaluations.list_evaluations_setups(         evaluation_measure,         flows=[flow_id],         tasks=[task_id],         size=limit_per_task,         output_format=\"dataframe\",     )      performance_column = \"value\"     # make a DataFrame consisting of all hyperparameters (which is a dict in setup['parameters']) and the performance     # value (in setup['value']). The following line looks a bit complicated, but combines 2 tasks: a) combine     # hyperparameters and performance data in a single dict, b) cast hyperparameter values to the appropriate format     # Note that the ``json.loads(...)`` requires the content to be in JSON format, which is only the case for     # scikit-learn setups (and even there some legacy setups might violate this requirement). It will work for the     # setups that belong to the flows embedded in this example though.     try:         setups_evals = pd.DataFrame(             [                 dict(                     **{name: json.loads(value) for name, value in setup[\"parameters\"].items()},                     **{performance_column: setup[performance_column]}                 )                 for _, setup in evals.iterrows()             ]         )     except json.decoder.JSONDecodeError as e:         print(\"Task %d error: %s\" % (task_id, e))         continue     # apply our filters, to have only the setups that comply to the hyperparameters we want     for filter_key, filter_value in parameter_filters.items():         setups_evals = setups_evals[setups_evals[filter_key] == filter_value]     # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters,     # the fanova library needs to be informed by using a configspace object.     setups_evals = setups_evals.select_dtypes(include=[\"int64\", \"float64\"])     # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``,     # ``verbose``.     setups_evals = setups_evals[         [             c             for c in list(setups_evals)             if len(setups_evals[c].unique()) &gt; 1 or c == performance_column         ]     ]     # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g.,     # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out:      # determine x values to pass to fanova library     parameter_names = [         pname for pname in setups_evals.columns.to_numpy() if pname != performance_column     ]     evaluator = fanova.fanova.fANOVA(         X=setups_evals[parameter_names].to_numpy(),         Y=setups_evals[performance_column].to_numpy(),         n_trees=n_trees,     )     for idx, pname in enumerate(parameter_names):         try:             fanova_results.append(                 {                     \"hyperparameter\": pname.split(\".\")[-1],                     \"fanova\": evaluator.quantify_importance([idx])[(idx,)][\"individual importance\"],                 }             )         except RuntimeError as e:             # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant             # for all configurations (there is no variance). We will skip these tasks (like the authors did in the             # paper).             print(\"Task %d error: %s\" % (task_id, e))             continue  # transform ``fanova_results`` from a list of dicts into a DataFrame fanova_results = pd.DataFrame(fanova_results) <p>make the boxplot of the variance contribution. Obviously, we can also use this data to make the Nemenyi plot, but this relies on the rather complex <code>Orange</code> dependency (<code>pip install Orange3</code>). For the complete example, the reader is referred to the more elaborate script (referred to earlier)</p> In\u00a0[\u00a0]: Copied! <pre>fig, ax = plt.subplots()\nsns.boxplot(x=\"hyperparameter\", y=\"fanova\", data=fanova_results, ax=ax)\nax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\nax.set_ylabel(\"Variance Contribution\")\nax.set_xlabel(None)\nplt.tight_layout()\nplt.show()\n# License: BSD 3-Clause\n</pre> fig, ax = plt.subplots() sns.boxplot(x=\"hyperparameter\", y=\"fanova\", data=fanova_results, ax=ax) ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\") ax.set_ylabel(\"Variance Contribution\") ax.set_xlabel(None) plt.tight_layout() plt.show() # License: BSD 3-Clause"},{"location":"examples/40_paper/2018_kdd_rijn_example/#van-rijn-and-hutter-2018","title":"van Rijn and Hutter (2018)\u00b6","text":"<p>A tutorial on how to reproduce the paper Hyperparameter Importance Across Datasets.</p> <p>This is a Unix-only tutorial, as the requirements can not be satisfied on a Windows machine (Untested on other systems).</p>"},{"location":"examples/40_paper/2018_kdd_rijn_example/#publication","title":"Publication\u00b6","text":"<p>| Hyperparameter importance across datasets | Jan N. van Rijn and Frank Hutter | In Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery &amp; Data Mining, 2018 | Available at https://dl.acm.org/doi/10.1145/3219819.3220058</p> <p>import sys</p> <p>if sys.platform == \"win32\":  # noqa print( \"The pyrfr library (requirement of fanova) can currently not be installed on Windows systems\" ) exit()</p> <p>import json import fanova import matplotlib.pyplot as plt import pandas as pd import seaborn as sns</p> <p>import openml</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/","title":"Perrone et al. (2018)","text":"In\u00a0[\u00a0]: Copied! <pre>import openml\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.ensemble import RandomForestRegressor\n\nflow_type = \"svm\"  # this example will use the smaller svm flow evaluations\n</pre> import openml import numpy as np import pandas as pd from matplotlib import pyplot as plt from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer from sklearn.compose import ColumnTransformer from sklearn.metrics import mean_squared_error from sklearn.preprocessing import OneHotEncoder from sklearn.ensemble import RandomForestRegressor  flow_type = \"svm\"  # this example will use the smaller svm flow evaluations <p>The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into a tabular format that can be used to build models.</p> In\u00a0[\u00a0]: Copied! <pre>def fetch_evaluations(run_full=False, flow_type=\"svm\", metric=\"area_under_roc_curve\"):\n    \"\"\"\n    Fetch a list of evaluations based on the flows and tasks used in the experiments.\n\n    Parameters\n    ----------\n    run_full : boolean\n        If True, use the full list of tasks used in the paper\n        If False, use 5 tasks with the smallest number of evaluations available\n    flow_type : str, {'svm', 'xgboost'}\n        To select whether svm or xgboost experiments are to be run\n    metric : str\n        The evaluation measure that is passed to openml.evaluations.list_evaluations\n\n    Returns\n    -------\n    eval_df : dataframe\n    task_ids : list\n    flow_id : int\n    \"\"\"\n    # Collecting task IDs as used by the experiments from the paper\n    # fmt: off\n    if flow_type == \"svm\" and run_full:\n        task_ids = [\n            10101, 145878, 146064, 14951, 34537, 3485, 3492, 3493, 3494,\n            37, 3889, 3891, 3899, 3902, 3903, 3913, 3918, 3950, 9889,\n            9914, 9946, 9952, 9967, 9971, 9976, 9978, 9980, 9983,\n        ]\n    elif flow_type == \"svm\" and not run_full:\n        task_ids = [9983, 3485, 3902, 3903, 145878]\n    elif flow_type == \"xgboost\" and run_full:\n        task_ids = [\n            10093, 10101, 125923, 145847, 145857, 145862, 145872, 145878,\n            145953, 145972, 145976, 145979, 146064, 14951, 31, 3485,\n            3492, 3493, 37, 3896, 3903, 3913, 3917, 3918, 3, 49, 9914,\n            9946, 9952, 9967,\n        ]\n    else:  # flow_type == 'xgboost' and not run_full:\n        task_ids = [3903, 37, 3485, 49, 3913]\n    # fmt: on\n\n    # Fetching the relevant flow\n    flow_id = 5891 if flow_type == \"svm\" else 6767\n\n    # Fetching evaluations\n    eval_df = openml.evaluations.list_evaluations_setups(\n        function=metric,\n        tasks=task_ids,\n        flows=[flow_id],\n        uploaders=[2702],\n        output_format=\"dataframe\",\n        parameters_in_separate_columns=True,\n    )\n    return eval_df, task_ids, flow_id\n\n\ndef create_table_from_evaluations(\n    eval_df, flow_type=\"svm\", run_count=np.iinfo(np.int64).max, task_ids=None\n):\n    \"\"\"\n    Create a tabular data with its ground truth from a dataframe of evaluations.\n    Optionally, can filter out records based on task ids.\n\n    Parameters\n    ----------\n    eval_df : dataframe\n        Containing list of runs as obtained from list_evaluations()\n    flow_type : str, {'svm', 'xgboost'}\n        To select whether svm or xgboost experiments are to be run\n    run_count : int\n        Maximum size of the table created, or number of runs included in the table\n    task_ids : list, (optional)\n        List of integers specifying the tasks to be retained from the evaluations dataframe\n\n    Returns\n    -------\n    eval_table : dataframe\n    values : list\n    \"\"\"\n    if task_ids is not None:\n        eval_df = eval_df[eval_df[\"task_id\"].isin(task_ids)]\n    if flow_type == \"svm\":\n        colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]\n    else:\n        colnames = [\n            \"alpha\",\n            \"booster\",\n            \"colsample_bylevel\",\n            \"colsample_bytree\",\n            \"eta\",\n            \"lambda\",\n            \"max_depth\",\n            \"min_child_weight\",\n            \"nrounds\",\n            \"subsample\",\n        ]\n    eval_df = eval_df.sample(frac=1)  # shuffling rows\n    eval_df = eval_df.iloc[:run_count, :]\n    eval_df.columns = [column.split(\"_\")[-1] for column in eval_df.columns]\n    eval_table = eval_df.loc[:, colnames]\n    value = eval_df.loc[:, \"value\"]\n    return eval_table, value\n\n\ndef list_categorical_attributes(flow_type=\"svm\"):\n    if flow_type == \"svm\":\n        return [\"kernel\"]\n    return [\"booster\"]\n</pre> def fetch_evaluations(run_full=False, flow_type=\"svm\", metric=\"area_under_roc_curve\"):     \"\"\"     Fetch a list of evaluations based on the flows and tasks used in the experiments.      Parameters     ----------     run_full : boolean         If True, use the full list of tasks used in the paper         If False, use 5 tasks with the smallest number of evaluations available     flow_type : str, {'svm', 'xgboost'}         To select whether svm or xgboost experiments are to be run     metric : str         The evaluation measure that is passed to openml.evaluations.list_evaluations      Returns     -------     eval_df : dataframe     task_ids : list     flow_id : int     \"\"\"     # Collecting task IDs as used by the experiments from the paper     # fmt: off     if flow_type == \"svm\" and run_full:         task_ids = [             10101, 145878, 146064, 14951, 34537, 3485, 3492, 3493, 3494,             37, 3889, 3891, 3899, 3902, 3903, 3913, 3918, 3950, 9889,             9914, 9946, 9952, 9967, 9971, 9976, 9978, 9980, 9983,         ]     elif flow_type == \"svm\" and not run_full:         task_ids = [9983, 3485, 3902, 3903, 145878]     elif flow_type == \"xgboost\" and run_full:         task_ids = [             10093, 10101, 125923, 145847, 145857, 145862, 145872, 145878,             145953, 145972, 145976, 145979, 146064, 14951, 31, 3485,             3492, 3493, 37, 3896, 3903, 3913, 3917, 3918, 3, 49, 9914,             9946, 9952, 9967,         ]     else:  # flow_type == 'xgboost' and not run_full:         task_ids = [3903, 37, 3485, 49, 3913]     # fmt: on      # Fetching the relevant flow     flow_id = 5891 if flow_type == \"svm\" else 6767      # Fetching evaluations     eval_df = openml.evaluations.list_evaluations_setups(         function=metric,         tasks=task_ids,         flows=[flow_id],         uploaders=[2702],         output_format=\"dataframe\",         parameters_in_separate_columns=True,     )     return eval_df, task_ids, flow_id   def create_table_from_evaluations(     eval_df, flow_type=\"svm\", run_count=np.iinfo(np.int64).max, task_ids=None ):     \"\"\"     Create a tabular data with its ground truth from a dataframe of evaluations.     Optionally, can filter out records based on task ids.      Parameters     ----------     eval_df : dataframe         Containing list of runs as obtained from list_evaluations()     flow_type : str, {'svm', 'xgboost'}         To select whether svm or xgboost experiments are to be run     run_count : int         Maximum size of the table created, or number of runs included in the table     task_ids : list, (optional)         List of integers specifying the tasks to be retained from the evaluations dataframe      Returns     -------     eval_table : dataframe     values : list     \"\"\"     if task_ids is not None:         eval_df = eval_df[eval_df[\"task_id\"].isin(task_ids)]     if flow_type == \"svm\":         colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]     else:         colnames = [             \"alpha\",             \"booster\",             \"colsample_bylevel\",             \"colsample_bytree\",             \"eta\",             \"lambda\",             \"max_depth\",             \"min_child_weight\",             \"nrounds\",             \"subsample\",         ]     eval_df = eval_df.sample(frac=1)  # shuffling rows     eval_df = eval_df.iloc[:run_count, :]     eval_df.columns = [column.split(\"_\")[-1] for column in eval_df.columns]     eval_table = eval_df.loc[:, colnames]     value = eval_df.loc[:, \"value\"]     return eval_table, value   def list_categorical_attributes(flow_type=\"svm\"):     if flow_type == \"svm\":         return [\"kernel\"]     return [\"booster\"] <p>Fetching the data from OpenML</p> <p>Now, we read all the tasks and evaluations for them and collate into a table. Here, we are reading all the tasks and evaluations for the SVM flow and pre-processing all retrieved evaluations.</p> In\u00a0[\u00a0]: Copied! <pre>eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type)\nX, y = create_table_from_evaluations(eval_df, flow_type=flow_type)\nprint(X.head())\nprint(\"Y : \", y[:5])\n</pre> eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type) X, y = create_table_from_evaluations(eval_df, flow_type=flow_type) print(X.head()) print(\"Y : \", y[:5]) In\u00a0[\u00a0]: Copied! <pre># Separating data into categorical and non-categorical (numeric for this example) columns\ncat_cols = list_categorical_attributes(flow_type=flow_type)\nnum_cols = list(set(X.columns) - set(cat_cols))\n\n# Missing value imputers for numeric columns\nnum_imputer = SimpleImputer(missing_values=np.nan, strategy=\"constant\", fill_value=-1)\n\n# Creating the one-hot encoder for numerical representation of categorical columns\nenc = OneHotEncoder(handle_unknown=\"ignore\")\n\n# Combining column transformers\nct = ColumnTransformer([(\"cat\", enc, cat_cols), (\"num\", num_imputer, num_cols)])\n\n# Creating the full pipeline with the surrogate model\nclf = RandomForestRegressor(n_estimators=50)\nmodel = Pipeline(steps=[(\"preprocess\", ct), (\"surrogate\", clf)])\n</pre> # Separating data into categorical and non-categorical (numeric for this example) columns cat_cols = list_categorical_attributes(flow_type=flow_type) num_cols = list(set(X.columns) - set(cat_cols))  # Missing value imputers for numeric columns num_imputer = SimpleImputer(missing_values=np.nan, strategy=\"constant\", fill_value=-1)  # Creating the one-hot encoder for numerical representation of categorical columns enc = OneHotEncoder(handle_unknown=\"ignore\")  # Combining column transformers ct = ColumnTransformer([(\"cat\", enc, cat_cols), (\"num\", num_imputer, num_cols)])  # Creating the full pipeline with the surrogate model clf = RandomForestRegressor(n_estimators=50) model = Pipeline(steps=[(\"preprocess\", ct), (\"surrogate\", clf)]) In\u00a0[\u00a0]: Copied! <pre># Selecting a task for the surrogate\ntask_id = task_ids[-1]\nprint(\"Task ID : \", task_id)\nX, y = create_table_from_evaluations(eval_df, task_ids=[task_id], flow_type=\"svm\")\n\nmodel.fit(X, y)\ny_pred = model.predict(X)\n\nprint(\"Training RMSE : {:.5}\".format(mean_squared_error(y, y_pred)))\n</pre> # Selecting a task for the surrogate task_id = task_ids[-1] print(\"Task ID : \", task_id) X, y = create_table_from_evaluations(eval_df, task_ids=[task_id], flow_type=\"svm\")  model.fit(X, y) y_pred = model.predict(X)  print(\"Training RMSE : {:.5}\".format(mean_squared_error(y, y_pred))) In\u00a0[\u00a0]: Copied! <pre># Sampling random configurations\ndef random_sample_configurations(num_samples=100):\n    colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]\n    ranges = [\n        (0.000986, 998.492437),\n        (2.0, 5.0),\n        (0.000988, 913.373845),\n        ([\"linear\", \"polynomial\", \"radial\", \"sigmoid\"]),\n    ]\n    X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames)\n    for i in range(len(colnames)):\n        if len(ranges[i]) == 2:\n            col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], size=num_samples)\n        else:\n            col_val = np.random.choice(ranges[i], size=num_samples)\n        X.iloc[:, i] = col_val\n    return X\n\n\nconfigs = random_sample_configurations(num_samples=1000)\nprint(configs)\n</pre> # Sampling random configurations def random_sample_configurations(num_samples=100):     colnames = [\"cost\", \"degree\", \"gamma\", \"kernel\"]     ranges = [         (0.000986, 998.492437),         (2.0, 5.0),         (0.000988, 913.373845),         ([\"linear\", \"polynomial\", \"radial\", \"sigmoid\"]),     ]     X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames)     for i in range(len(colnames)):         if len(ranges[i]) == 2:             col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], size=num_samples)         else:             col_val = np.random.choice(ranges[i], size=num_samples)         X.iloc[:, i] = col_val     return X   configs = random_sample_configurations(num_samples=1000) print(configs) In\u00a0[\u00a0]: Copied! <pre>preds = model.predict(configs)\n\n# tracking the maximum AUC obtained over the functions evaluations\npreds = np.maximum.accumulate(preds)\n# computing regret (1 - predicted_auc)\nregret = 1 - preds\n\n# plotting the regret curve\nplt.plot(regret)\nplt.title(\"AUC regret for Random Search on surrogate\")\nplt.xlabel(\"Numbe of function evaluations\")\nplt.ylabel(\"Regret\")\n# License: BSD 3-Clause\n</pre> preds = model.predict(configs)  # tracking the maximum AUC obtained over the functions evaluations preds = np.maximum.accumulate(preds) # computing regret (1 - predicted_auc) regret = 1 - preds  # plotting the regret curve plt.plot(regret) plt.title(\"AUC regret for Random Search on surrogate\") plt.xlabel(\"Numbe of function evaluations\") plt.ylabel(\"Regret\") # License: BSD 3-Clause"},{"location":"examples/40_paper/2018_neurips_perrone_example/#perrone-et-al-2018","title":"Perrone et al. (2018)\u00b6","text":"<p>A tutorial on how to build a surrogate model based on OpenML data as done for Scalable Hyperparameter Transfer Learning by Perrone et al..</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#publication","title":"Publication\u00b6","text":"<p>| Scalable Hyperparameter Transfer Learning | Valerio Perrone and Rodolphe Jenatton and Matthias Seeger and Cedric Archambeau | In Advances in Neural Information Processing Systems 31, 2018 | Available at https://papers.nips.cc/paper/7917-scalable-hyperparameter-transfer-learning.pdf</p> <p>This example demonstrates how OpenML runs can be used to construct a surrogate model.</p> <p>In the following section, we shall do the following:</p> <ul> <li>Retrieve tasks and flows as used in the experiments by Perrone et al. (2018).</li> <li>Build a tabular data by fetching the evaluations uploaded to OpenML.</li> <li>Impute missing values and handle categorical data before building a Random Forest model that maps hyperparameter values to the area under curve score.</li> </ul>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#creating-pre-processing-and-modelling-pipelines","title":"Creating pre-processing and modelling pipelines\u00b6","text":"<p>The two primary tasks are to impute the missing values, that is, account for the hyperparameters that are not available with the runs from OpenML. And secondly, to handle categorical variables using One-hot encoding prior to modelling.</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#building-a-surrogate-model-on-a-tasks-evaluation","title":"Building a surrogate model on a task's evaluation\u00b6","text":"<p>The same set of functions can be used for a single task to retrieve a singular table which can be used for the surrogate model construction. We shall use the SVM flow here to keep execution time simple and quick.</p>"},{"location":"examples/40_paper/2018_neurips_perrone_example/#evaluating-the-surrogate-model","title":"Evaluating the surrogate model\u00b6","text":"<p>The surrogate model built from a task's evaluations fetched from OpenML will be put into trivial action here, where we shall randomly sample configurations and observe the trajectory of the area under curve (auc) we can obtain from the surrogate we've built.</p> <p>NOTE: This section is written exclusively for the SVM flow</p>"},{"location":"help/","title":"Questions","text":"<p>If you have questions, you can contact us in various ways. Here are the best ways to reach out depending on your questions.</p>"},{"location":"help/#general-questions","title":"General questions","text":"<p>For all non-technical questions, such as how to get involved, collaboration ideas, and interesting proposals, please should the communication channel of your choice. We believe in making the world better, together, so we're always happy to hear from you </p> <ul> <li> <p> Contact us via email</p> <p>Old school. We love it :). This email reaches all the main OpenML developers.</p> <p> Write an email</p> </li> <li> <p> Chat with us on Slack</p> <p>Get involved in current discussions or start your own.</p> <p> Join us on Slack</p> </li> <li> <p> Social media</p> <p>You can reach us on X (formerly Twitter)</p> <p> Post something now</p> </li> </ul>"},{"location":"help/#technical-questions","title":"Technical questions","text":"<p>For technical questions, the best way is to open an issue on GitHub. We have several issue trackers which are closely monitored by the people who can best answer your questions. We are happy to help, so don't be shy </p>"},{"location":"help/#openml-client-libraries","title":"OpenML client libraries","text":"<ul> <li> <p> Using OpenML in Python</p> <p>For all questions on using OpenML via the Python API</p> <p> Post your question</p> </li> <li> <p> Using OpenML in R</p> <p>For all questions on using OpenML via the R API (via mlr3)</p> <p> Post your question</p> </li> <li> <p> Using OpenML in Julia</p> <p>For all questions on using OpenML via the Julia API</p> <p> Post your question</p> </li> <li> <p> Using OpenML in Java</p> <p>For all questions on using OpenML via the Java API</p> <p> Post your question</p> </li> </ul>"},{"location":"help/#openml-platform","title":"OpenML platform","text":"<ul> <li> <p> OpenML Server</p> <p>For all questions on the OpenML server and REST API</p> <p> Version 1 of the API (PHP)  Version 2 of the API (Python)</p> </li> <li> <p> OpenML Website</p> <p>For all questions about the OpenML website</p> <p> Post your question</p> </li> <li> <p> OpenML deployment</p> <p>For all questions on deploying OpenML locally or how OpenML runs in production</p> <p> Post your question</p> </li> <li> <p> OpenML documentation</p> <p>For all questions and suggestions for these documentation pages</p> <p> Post your question</p> </li> </ul>"},{"location":"help/#openml-content","title":"OpenML content","text":"<ul> <li> <p> Datasets</p> <p>For all questions about OpenML datasets</p> <p> Post your question</p> </li> <li> <p> Croissant</p> <p>For all questions about Croissant, the metadata standard we use for describing machine learning datasets</p> <p> Post your question</p> </li> </ul>"},{"location":"intro/Governance/","title":"Governance","text":"<p>The purpose of this document is to formalize the governance process used by the OpenML project (the OpenML GitHub organization which contains all code and projects related to OpenML.org), to clarify how decisions are made and how the various elements of our community interact. This document establishes a decision-making structure that takes into account feedback from all members of the community and strives to find consensus, while avoiding any deadlocks.</p> <p>The OpenML project is an independent open source project that is legally represented by the Open Machine Learning Foundation. The Open Machine Learning Foundation is a not-for-profit organization supporting, but not controlling, the OpenML project. The Foundation is open to engage with universities, companies, or anyone sharing the same goals. The OpenML project has a separate governance model described in this document.</p> <p>This is a meritocratic, consensus-based community project. Anyone with an interest in the project can join the community, contribute to the project design, and participate in the decision making process. This document describes how that participation takes place and how to set about earning merit within the project community.</p>"},{"location":"intro/Governance/#roles-and-responsibilities","title":"Roles And Responsibilities","text":""},{"location":"intro/Governance/#contributors","title":"Contributors","text":"<p>Contributors are community members who contribute in concrete ways to the project. Anyone can become a contributor, and contributions can take many forms \u2013 not only code \u2013 as detailed in the contributors guide. Contributors need to create pull requests to contribute to the code or documentation.</p>"},{"location":"intro/Governance/#core-contributors","title":"Core contributors","text":"<p>Core contributors are community members who have shown that they are dedicated to the continued development of the project through ongoing engagement with the community. They have shown they can be trusted to maintain OpenML with care. Being a core contributor allows contributors to more easily carry on with their project related activities by giving them write access to the project\u2019s repository (abiding by the decision making process described below, e.g. merging pull requests that obey the decision making procedure described below) and is represented as being an organization member on the OpenML GitHub organization. Core contributors are expected to review code contributions, can merge approved pull requests, can cast votes for and against merging a pull-request, and can be involved in deciding major changes to the API.</p> <p>New core contributors can be nominated by any existing core contributors. Once they have been nominated, there will be a vote in the private OpenML core email list by the current core contributors. While it is expected that most votes will be unanimous, a two-thirds majority of the cast votes is enough. The vote needs to be open for at least 1 week.</p> <p>Core contributors that have not contributed to the project (commits or GitHub comments) in the past 12 months will become emeritus core contributors and recant their commit and voting rights until they become active again. The list of core contributors, active and emeritus (with dates at which they became active) is public on the OpenML website.</p>"},{"location":"intro/Governance/#steering-committee","title":"Steering Committee","text":"<p>The Steering Committee (SC) members are core contributors who have additional responsibilities to ensure the smooth running of the project. SC members are expected to participate in strategic planning, join monthly meetings, and approve changes to the governance model. The purpose of the SC is to ensure a smooth progress from the big-picture perspective. Indeed, changes that impact the full project require a synthetic analysis and a consensus that is both explicit and informed. In cases that the core contributor community (which includes the SC members) fails to reach such a consensus in the required time frame, the SC is the entity to resolve the issue.</p> <p>The SC consists of community representatives and partner representatives. Community representatives of the SC are nominated by a core contributor. A nomination will result in a discussion that cannot take more than a month and then a vote by the core contributors which will stay open for a week. SC membership votes are subject to a two-third majority of all cast votes as well as a simple majority approval of all the current SC members.</p> <p>Partner institutions who enter a collaboration agreement or sponsorship agreement with the OpenML Foundation can nominate a representative on the Steering Committee, if so agreed in the agreement. Such a collaboration should in principle include one full-time developer to work on OpenML (in cash or in kind) for the duration of the agreement. New partner representatives have to be confirmed by the SC following the same voting rules above.</p> <p>The OpenML community must have at least equal footing in the steering committee. Additional SC members may be nominated to ensure this, following the membership voting rules described above.</p> <p>When decisions are escalated to the steering committee (see the decision making process below), and no consensus can be found within a month, the SC can meet and decide by consensus or with a simple majority of all cast votes.</p> <p>SC members who do not actively engage with the SC duties are expected to resign.</p> <p>The initial Steering Committee of OpenML consists of Bernd Bischl, Giuseppe Casalicchio, Matthias Feurer, Heidi Seibold, Jan van Rijn, and Joaquin Vanschoren. They all represent the OpenML community.</p>"},{"location":"intro/Governance/#decision-making-process","title":"Decision Making Process","text":"<p>Decisions about the future of the project are made through discussion with all members of the community. All non-sensitive project management discussion takes place on the project contributors\u2019 mailing list and the issue trackers of the sub-projects. Occasionally, sensitive discussion occurs on the private core developer email list (see below). This includes voting on core/SC membership or discussion of internal disputes. All discussions must follow the OpenML honor code.</p> <p>OpenML uses a \u201cconsensus seeking\u201d process for making decisions. The group tries to find a resolution that has no open objections among core contributors. At any point during the discussion, any core contributors can call for a vote, which will conclude one month from the call for the vote, or when two thirds of all votes are in favor.</p> <p>If no option can gather two thirds of the votes cast (ignoring abstentions), the decision is escalated to the SC, which in turn will use consensus seeking with the fallback option of a simple majority vote if no consensus can be found within a month. This is what we hereafter may refer to as \u201cthe decision making process\u201d. It applies to all core OpenML repositories.</p> <p>Decisions (in addition to adding core contributors and SC membership as above) are made according to the following rules:</p> <p>Normal changes:  </p> <ul> <li>Minor Documentation changes, such as typo fixes, or addition / correction of a sentence: requires one approved review by a core contributor, and no objections in the comments (lazy consensus). Core contributors are expected to give \u201creasonable time\u201d to others to give their opinion on the pull request if they\u2019re not confident others would agree.  </li> <li>Non-server packages that only have one core contributor are not subject to the ruling in the bullet point above (i.e. a sole core developer can make decisions on their own).</li> </ul> <p>Major changes:  </p> <ul> <li>Major changes to the API principles and metadata schema require a concrete proposal outlined in an OpenML Request for Comments (RfC), which has to be opened for public consultation for at least 1 month. The final version has to be approved using the decision-making process outlined above (two-third of the cast vote by core contributors or simple majority if escalated to the SC). Voting is typically done as a comment in the pull request (+1, -1, or 0 to abstain).  </li> <li>RfCs must be announced and shared via the public mailing list and may link additional content (such as blog posts or google docs etc. detailing the changes).  </li> <li>Changes to the governance model use the same decision process outlined above.  </li> </ul> <p>If a veto -1 vote is cast on a lazy consensus, the proposer can appeal to the community and core contributors and the change can be approved or rejected using the decision making procedure outlined above.</p>"},{"location":"intro/Governance/#communication-channels","title":"Communication channels","text":"<p>OpenML uses the following communication channels:  </p> <ul> <li>An open contributor mailing list and the GitHub issue trackers.  </li> <li>A chat application for daily interaction with the community (currently Slack).  </li> <li>Private email lists (without archive) for the core developers (core@openml.org) and steering committee (steering@openml.org), for membership voting and sensitive discussions.  </li> <li>Biyearly Steering Committee meeting at predefined times, listed on the website, and asynchronous discussions on a discussion board. They are open to all steering committee members and core contributors, and they can all request discussion on a topic. Closed meetings for SC members only can be called in if there are sensitive discussions or other valid reasons.  </li> <li>A monthly Engineering meeting at predefined times, listed on the website. The meeting is open to all. Discussion points are put on the [project roadmap]  (https://github.com/orgs/openml/projects/2).</li> </ul>"},{"location":"intro/showcase/","title":"Research using OpenML","text":"<p>This page will have a list of interesting research papers that have used OpenML. If you have used OpenML in your research and would like to have your paper listed here, please drop a PR with the relevant information (click the  icon above).</p>"},{"location":"intro/terms/","title":"Terms","text":""},{"location":"intro/terms/#honor-code","title":"Honor Code","text":"<p>By joining OpenML, you join a special worldwide community of data scientists building on each other's results and connecting their minds as efficiently as possible. This community depends on your motivation to share data, tools and ideas, and to do so with honesty. In return, you will gain trust, visibility and reputation, igniting online collaborations and studies that otherwise may not have happened.</p> <p>By using any part of OpenML, you agree to:</p> <ul> <li>Give credit where credit is due. Cite the authors whose work you are building on, or build collaborations where appropriate.</li> <li>Give back to the community by sharing your own data as openly and as soon as possible, or by helping the community in other ways. In doing so, you gain visibility and impact (citations).</li> <li>Share data according to your best efforts. Everybody make mistakes, but we trust you to correct them as soon as possible. Remove or flag data that cannot be trusted.</li> <li>Be polite and constructive in all discussions. Criticism of methods is welcomed, but personal criticisms should be avoided.</li> <li>Respect circles of trust. OpenML allows you to collaborate in 'circles' of trusted people to share unpublished results. Be considerate in sharing data with people outside this circle.</li> <li>Do not steal the work of people who openly share it. OpenML makes it easy to find all shared data (and when it was shared), thus everybody will know if you do this.</li> </ul>"},{"location":"intro/terms/#terms-of-use","title":"Terms of Use","text":"<p>You agree that you are responsible for your own use of OpenML.org and all content submitted by you, in accordance with the Honor Code and all applicable local, state, national and international laws.</p> <p>By submitting or distributing content from OpenML.org, you affirm that you have the necessary rights, licenses, consents and/or permissions to reproduce and publish this content. You cannot upload sensitive or confidential data. You, and not the developers of OpenML.org, are solely responsible for your submissions.</p> <p>By submitting content to OpenML.org, you grant OpenML.org the right to host, transfer, display and use this content, in accordance with your sharing settings and any licences granted by you. You also grant to each user a non-exclusive license to access and use this content for their own research purposes, in accordance with any licences granted by you.</p> <p>You may maintain one user account and not let anyone else use your username and/or password. You may not impersonate other persons.</p> <p>You will not intend to damage, disable, or impair any OpenML server or interfere with any other party's use and enjoyment of the service. You may not attempt to gain unauthorized access to the Site, other accounts, computer systems or networks connected to any OpenML server. You may not obtain or attempt to obtain any materials or information not intentionally made available through OpenML.</p> <p>Strictly prohibited are content that defames, harasses or threatens others, that infringes another's intellectual property, as well as indecent or unlawful content, advertising, or intentionally inaccurate information posted with the intent of misleading others. It is also prohibited to post code containing viruses, malware, spyware or any other similar software that may damage the operation of another's computer or property.</p>"},{"location":"julia/","title":"Home","text":""},{"location":"julia/#openmljl-documentation","title":"OpenML.jl Documentation","text":"<p>This is the reference documentation of <code>OpenML.jl</code>.</p> <p>The OpenML platform provides an integration platform for carrying out and comparing machine learning solutions across a broad collection of public datasets and software platforms.</p> <p>Summary of OpenML.jl functionality:</p> <ul> <li><code>OpenML.list_tags</code><code>()</code>: for listing all dataset tags</li> <li><code>OpenML.list_datasets</code><code>(; tag=nothing, filter=nothing, output_format=...)</code>: for listing available datasets</li> <li><code>OpenML.describe_dataset</code><code>(id)</code>: to describe a particular dataset</li> <li><code>OpenML.load</code><code>(id; parser=:arff)</code>: to download a dataset</li> </ul> <p></p> <p></p>"},{"location":"julia/#installation","title":"Installation","text":"<pre><code>using Pkg\nPkg.add(\"OpenML\")\n</code></pre> <p>If running the demonstration below:</p> <pre><code>Pkg.add(\"DataFrames\") \nPkg.add(\"ScientificTypes\")\n</code></pre> <p></p> <p></p>"},{"location":"julia/#sample-usage","title":"Sample usage","text":"<pre><code>julia&gt; using OpenML # or using MLJ\n\n\njulia&gt; using DataFrames\n\n\njulia&gt; OpenML.list_tags()\n300-element Vector{Any}:\n \"study_41\"\n \"uci\"\n \"study_34\"\n \"study_37\"\n \"mythbusting_1\"\n \"OpenML-CC18\"\n \"study_99\"\n \"artificial\"\n \"BNG\"\n \"study_16\"\n \u22ee\n \"Earth Science\"\n \"Social Media\"\n \"Meteorology\"\n \"Geography\"\n \"Language\"\n \"Computational Universe\"\n \"History\"\n \"Culture\"\n \"Sociology\"\n</code></pre> <p>Listing all datasets with the \"OpenML100\" tag which also have <code>n</code> instances and <code>p</code> features, where <code>100 &lt; n &lt; 1000</code> and <code>1 &lt; p &lt; 10</code>:</p> <pre><code>julia&gt; ds = OpenML.list_datasets(\n                 tag = \"OpenML100\",\n                 filter = \"number_instances/100..1000/number_features/1..10\",\n                 output_format = DataFrame)\n12\u00d713 DataFrame\n Row \u2502 id     name                              status  MajorityClassSize  Max \u22ef\n     \u2502 Int64  String                            String  Int64?             Int \u22ef\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502    11  balance-scale                     active                288      \u22ef\n   2 \u2502    15  breast-w                          active                458\n   3 \u2502    37  diabetes                          active                500\n   4 \u2502    50  tic-tac-toe                       active                626\n   5 \u2502   333  monks-problems-1                  active                278      \u22ef\n   6 \u2502   334  monks-problems-2                  active                395\n   7 \u2502   335  monks-problems-3                  active                288\n   8 \u2502   451  irish                             active                278\n   9 \u2502   469  analcatdata_dmft                  active                155      \u22ef\n  10 \u2502   470  profb                             active                448\n  11 \u2502  1464  blood-transfusion-service-center  active                570\n  12 \u2502 40496  LED-display-domain-7digit         active                 57\n                                                               9 columns omitted\n</code></pre> <p>Describing and loading one of these datasets:</p> <pre><code>julia&gt; OpenML.describe_dataset(15)\n  Author: Dr. William H. Wolberg, University of Wisconsin Source: UCI\n  (https://archive.ics.uci.edu/ml/datasets/breast+cancer+wisconsin+(original)),\n  University of Wisconsin (http://pages.cs.wisc.edu/~olvi/uwmp/cancer.html) -\n  1995 Please cite: See below, plus UCI\n  (https://archive.ics.uci.edu/ml/citation_policy.html)\n\n  Breast Cancer Wisconsin (Original) Data Set. Features are computed from a\n  digitized image of a fine needle aspirate (FNA) of a breast mass. They\n  describe characteristics of the cell nuclei present in the image. The target\n  feature records the prognosis (malignant or benign). Original data available\n  here (ftp://ftp.cs.wisc.edu/math-prog/cpo-dataset/machine-learn/cancer/)\n\n  Current dataset was adapted to ARFF format from the UCI version. Sample code\n  ID's were removed.\n\n  ! Note that there is also a related Breast Cancer Wisconsin (Diagnosis) Data\n  Set with a different set of features, better known as wdbc\n  (https://www.openml.org/d/1510).\n\n  Relevant Papers\n  \u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\n\n  W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction\n  for breast tumor diagnosis. IS&amp;T/SPIE 1993 International Symposium on\n  Electronic Imaging: Science and Technology, volume 1905, pages 861-870, San\n  Jose, CA, 1993.\n\n  O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and\n  prognosis via linear programming. Operations Research, 43(4), pages 570-577,\n  July-August 1995.\n\n  Citation request\n  \u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\n\n  This breast cancer database was obtained from the University of Wisconsin\n  Hospitals, Madison from Dr. William H. Wolberg. If you publish results when\n  using this database, then please include this information in your\n  acknowledgments. Also, please cite one or more of:\n\n    1. O. L. Mangasarian and W. H. Wolberg: \"Cancer diagnosis via linear\n       programming\", SIAM News, Volume 23, Number 5, September 1990, pp 1\n       &amp; 18.\n\n    2. William H. Wolberg and O.L. Mangasarian: \"Multisurface method of\n       pattern separation for medical diagnosis applied to breast\n       cytology\", Proceedings of the National Academy of Sciences,\n       U.S.A., Volume 87, December 1990, pp 9193-9196.\n\n    3. O. L. Mangasarian, R. Setiono, and W.H. Wolberg: \"Pattern\n       recognition via linear programming: Theory and application to\n       medical diagnosis\", in: \"Large-scale numerical optimization\",\n       Thomas F. Coleman and Yuying Li, editors, SIAM Publications,\n       Philadelphia 1990, pp 22-30.\n\n    4. K. P. Bennett &amp; O. L. Mangasarian: \"Robust linear programming\n       discrimination of two linearly inseparable sets\", Optimization\n       Methods and Software 1, 1992, 23-34 (Gordon &amp; Breach Science\n       Publishers).\n\njulia&gt; table = OpenML.load(15)\nTables.DictColumnTable with 699 rows, 10 columns, and schema:\n :Clump_Thickness        Float64\n :Cell_Size_Uniformity   Float64\n :Cell_Shape_Uniformity  Float64\n :Marginal_Adhesion      Float64\n :Single_Epi_Cell_Size   Float64\n :Bare_Nuclei            Union{Missing, Float64}\n :Bland_Chromatin        Float64\n :Normal_Nucleoli        Float64\n :Mitoses                Float64\n :Class                  CategoricalArrays.CategoricalValue{String, UInt32}\n</code></pre> <p>Converting to a data frame:</p> <pre><code>julia&gt; df = DataFrame(table)\n699\u00d710 DataFrame\n Row \u2502 Clump_Thickness  Cell_Size_Uniformity  Cell_Shape_Uniformity  Marginal_ \u22ef\n     \u2502 Float64          Float64               Float64                Float64   \u22ef\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502             5.0                   1.0                    1.0            \u22ef\n   2 \u2502             5.0                   4.0                    4.0\n   3 \u2502             3.0                   1.0                    1.0\n   4 \u2502             6.0                   8.0                    8.0\n   5 \u2502             4.0                   1.0                    1.0            \u22ef\n   6 \u2502             8.0                  10.0                   10.0\n   7 \u2502             1.0                   1.0                    1.0\n   8 \u2502             2.0                   1.0                    2.0\n  \u22ee  \u2502        \u22ee                  \u22ee                      \u22ee                    \u22ee \u22f1\n 693 \u2502             3.0                   1.0                    1.0            \u22ef\n 694 \u2502             3.0                   1.0                    1.0\n 695 \u2502             3.0                   1.0                    1.0\n 696 \u2502             2.0                   1.0                    1.0\n 697 \u2502             5.0                  10.0                   10.0            \u22ef\n 698 \u2502             4.0                   8.0                    6.0\n 699 \u2502             4.0                   8.0                    8.0\n                                                  7 columns and 684 rows omitted\n</code></pre> <p>Inspecting it's schema:</p> <pre><code>julia&gt; using ScientificTypes\n\n\njulia&gt; schema(table)\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2502 names                 \u2502 scitypes                   \u2502 types                   \u22ef\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2502 Clump_Thickness       \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Cell_Size_Uniformity  \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Cell_Shape_Uniformity \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Marginal_Adhesion     \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Single_Epi_Cell_Size  \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Bare_Nuclei           \u2502 Union{Missing, Continuous} \u2502 Union{Missing, Float64} \u22ef\n\u2502 Bland_Chromatin       \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Normal_Nucleoli       \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Mitoses               \u2502 Continuous                 \u2502 Float64                 \u22ef\n\u2502 Class                 \u2502 Multiclass{2}              \u2502 CategoricalValue{String \u22ef\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n                                                                1 column omitted\n</code></pre> <p></p> <p></p>"},{"location":"julia/#public-api","title":"Public API","text":""},{"location":"julia/#openmllist_tags","title":"<code>OpenML.list_tags</code>","text":"<pre><code>list_tags()\n</code></pre> <p>List all available tags.</p>"},{"location":"julia/#openmllist_datasets","title":"<code>OpenML.list_datasets</code>","text":"<pre><code>list_datasets(; tag = nothing, filters = \"\", output_format = NamedTuple)\n</code></pre> <p>Lists all active OpenML datasets, if <code>tag = nothing</code> (default). To list only datasets with a given tag, choose one of the tags in <code>list_tags()</code>. An alternative <code>output_format</code> can be chosen, e.g. <code>DataFrame</code>, if the <code>DataFrames</code> package is loaded.</p> <p>A filter is a string of <code>&lt;data quality&gt;/&lt;range&gt;</code> or <code>&lt;data quality&gt;/&lt;value&gt;</code> pairs, concatenated using <code>/</code>, such as</p> <pre><code>    filter = \"number_features/10/number_instances/500..10000\"\n</code></pre> <p>The allowed data qualities include <code>tag</code>, <code>status</code>, <code>limit</code>, <code>offset</code>, <code>data_id</code>, <code>data_name</code>, <code>data_version</code>, <code>uploader</code>, <code>number_instances</code>, <code>number_features</code>, <code>number_classes</code>, <code>number_missing_values</code>.</p> <p>For more on the format and effect of <code>filters</code> refer to the openml API.</p> <p>Examples</p> <pre><code>julia&gt; using DataFrames\n\njulia&gt; ds = OpenML.list_datasets(\n               tag = \"OpenML100\",\n               filter = \"number_instances/100..1000/number_features/1..10\",\n               output_format = DataFrame\n)\n\njulia&gt; sort!(ds, :NumberOfFeatures)\n</code></pre>"},{"location":"julia/#openmldescribe_dataset","title":"<code>OpenML.describe_dataset</code>","text":"<pre><code>describe_dataset(id)\n</code></pre> <p>Load and show the OpenML description of the data set <code>id</code>. Use <code>list_datasets</code> to browse available data sets.</p> <p>Examples</p> <pre><code>julia&gt; OpenML.describe_dataset(6)\n  Author: David J. Slate Source: UCI\n  (https://archive.ics.uci.edu/ml/datasets/Letter+Recognition) - 01-01-1991 Please cite: P.\n  W. Frey and D. J. Slate. \"Letter Recognition Using Holland-style Adaptive Classifiers\".\n  Machine Learning 6(2), 1991\n\n    1. TITLE:\n\n  Letter Image Recognition Data\n\n  The objective is to identify each of a large number of black-and-white\n  rectangular pixel displays as one of the 26 capital letters in the English\n  alphabet.  The character images were based on 20 different fonts and each\n  letter within these 20 fonts was randomly distorted to produce a file of\n  20,000 unique stimuli.  Each stimulus was converted into 16 primitive\n  numerical attributes (statistical moments and edge counts) which were then\n  scaled to fit into a range of integer values from 0 through 15.  We\n  typically train on the first 16000 items and then use the resulting model\n  to predict the letter category for the remaining 4000.  See the article\n  cited above for more details.\n</code></pre>"},{"location":"julia/#openmlload","title":"<code>OpenML.load</code>","text":"<pre><code>OpenML.load(id; maxbytes = nothing)\n</code></pre> <p>Load the OpenML dataset with specified <code>id</code>, from those listed by <code>list_datasets</code> or on the OpenML site.</p> <p>Datasets are saved as julia artifacts so that they persist locally once loaded.</p> <p>Returns a table.</p> <p>Examples</p> <pre><code>using DataFrames\ntable = OpenML.load(61)\ndf = DataFrame(table) # transform to a DataFrame\nusing ScientificTypes\ndf2 = coerce(df, autotype(df)) # corce to automatically detected scientific types\n\npeek_table = OpenML.load(61, maxbytes = 1024) # load only the first 1024 bytes of the table\n</code></pre>"},{"location":"notebooks/getting_started/","title":"Getting Started","text":"In\u00a0[2]: Copied! <pre>!pip install -q openml\n</pre> !pip install -q openml In\u00a0[3]: Copied! <pre># License: BSD 3-Clause\n\nimport openml\nfrom sklearn import neighbors\n</pre> # License: BSD 3-Clause  import openml from sklearn import neighbors In\u00a0[4]: Copied! <pre>openml.config.start_using_configuration_for_example()\n</pre> openml.config.start_using_configuration_for_example() <pre>/var/folders/0t/5d8ttqzd773fy0wq3h5db0xr0000gn/T/ipykernel_60921/256497051.py:1: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!\n  openml.config.start_using_configuration_for_example()\n</pre> <p>When using the main server instead, make sure your apikey is configured. This can be done with the following line of code (uncomment it!). Never share your apikey with others.</p> In\u00a0[5]: Copied! <pre># openml.config.apikey = 'YOURKEY'\n</pre> # openml.config.apikey = 'YOURKEY' In\u00a0[6]: Copied! <pre># Uncomment and set your OpenML cache directory\n# import os\n# openml.config.cache_directory = os.path.expanduser('YOURDIR')\n</pre> # Uncomment and set your OpenML cache directory # import os # openml.config.cache_directory = os.path.expanduser('YOURDIR') In\u00a0[7]: Copied! <pre>task = openml.tasks.get_task(403)\ndata = openml.datasets.get_dataset(task.dataset_id)\nclf = neighbors.KNeighborsClassifier(n_neighbors=5)\nrun = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n# Publish the experiment on OpenML (optional, requires an API key).\n# For this tutorial, our configuration publishes to the test server\n# as to not crowd the main server with runs created by examples.\nmyrun = run.publish()\nprint(f\"kNN on {data.name}: {myrun.openml_url}\")\n</pre> task = openml.tasks.get_task(403) data = openml.datasets.get_dataset(task.dataset_id) clf = neighbors.KNeighborsClassifier(n_neighbors=5) run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False) # Publish the experiment on OpenML (optional, requires an API key). # For this tutorial, our configuration publishes to the test server # as to not crowd the main server with runs created by examples. myrun = run.publish() print(f\"kNN on {data.name}: {myrun.openml_url}\") <pre>kNN on eeg-eye-state: https://test.openml.org/r/32906\n</pre> In\u00a0[8]: Copied! <pre>openml.config.stop_using_configuration_for_example()\n</pre> openml.config.stop_using_configuration_for_example()"},{"location":"notebooks/getting_started/#getting-started","title":"Getting Started\u00b6","text":"<p>This page will guide you through the process of getting started with OpenML. While this page is a good starting point, for more detailed information, please refer to the integrations section and the rest of the documentation.</p>"},{"location":"notebooks/getting_started/#authentication","title":"Authentication\u00b6","text":"<ul> <li>If you are using the OpenML API to download datasets, upload results, or create tasks, you will need to authenticate. You can do this by creating an account on the OpenML website and using your API key. - You can find detailed instructions on how to authenticate in the authentication section</li> </ul>"},{"location":"notebooks/getting_started/#eeg-eye-state-example","title":"EEG Eye State example\u00b6","text":"<p>Download the OpenML task for the eeg-eye-state.</p>"},{"location":"notebooks/getting_started/#caching","title":"Caching\u00b6","text":"<p>When downloading datasets, tasks, runs and flows, they will be cached to retrieve them without calling the server later. As with the API key, the cache directory can be either specified through the config file or through the API:</p> <ul> <li>Add the  line cachedir = 'MYDIR' to the config file, replacing 'MYDIR' with the path to the cache directory. By default, OpenML will use ~/.openml/cache as the cache directory.</li> <li>Run the code below, replacing 'YOURDIR' with the path to the cache directory.</li> </ul>"},{"location":"python/","title":"OpenML-Python","text":"<p>A python interface for OpenML, an online platform for open science collaboration in machine learning. It can be used to download or upload OpenML data such as datasets and machine learning experiment results.</p>"},{"location":"python/#general","title":"General","text":"<ul> <li>Documentation.</li> <li>Contribution guidelines.</li> </ul>"},{"location":"python/#citing-openml-python","title":"Citing OpenML-Python","text":"<p>If you use OpenML-Python in a scientific publication, we would appreciate a reference to the following paper:</p> <p>Matthias Feurer, Jan N. van Rijn, Arlind Kadra, Pieter Gijsbers, Neeratyoy Mallik, Sahithya Ravi, Andreas M\u00fcller, Joaquin Vanschoren, Frank Hutter OpenML-Python: an extensible Python API for OpenML Journal of Machine Learning Research, 22(100):1\u22125, 2021</p> <p>Bibtex entry: <pre><code>@article{JMLR:v22:19-920,\n  author  = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas M\u00fcller and Joaquin Vanschoren and Frank Hutter},\n  title   = {OpenML-Python: an extensible Python API for OpenML},\n  journal = {Journal of Machine Learning Research},\n  year    = {2021},\n  volume  = {22},\n  number  = {100},\n  pages   = {1--5},\n  url     = {http://jmlr.org/papers/v22/19-920.html}\n}\n</code></pre></p>"},{"location":"python/#contributors","title":"Contributors \u2728","text":"<p>Thanks goes to these wonderful people (emoji key):</p> <sub>a-moadel</sub>\ud83d\udcd6 \ud83d\udca1 <sub>Neeratyoy Mallik</sub>\ud83d\udcbb \ud83d\udcd6 \ud83d\udca1 <p>This project follows the all-contributors specification. Contributions of any kind welcome!</p>"},{"location":"python/","title":"OpenML","text":"<p>Collaborative Machine Learning in Python</p> <p>Welcome to the documentation of the OpenML Python API, a connector to the collaborative machine learning platform OpenML.org. The OpenML Python package allows to use datasets and tasks from OpenML together with scikit-learn and share the results online.</p>"},{"location":"python/#example","title":"Example","text":"<pre><code>import openml\nfrom sklearn import impute, tree, pipeline\n\n# Define a scikit-learn classifier or pipeline\nclf = pipeline.Pipeline(\n    steps=[\n        ('imputer', impute.SimpleImputer()),\n        ('estimator', tree.DecisionTreeClassifier())\n    ]\n)\n# Download the OpenML task for the pendigits dataset with 10-fold\n# cross-validation.\ntask = openml.tasks.get_task(32)\n# Run the scikit-learn model on the task.\nrun = openml.runs.run_model_on_task(clf, task)\n# Publish the experiment on OpenML (optional, requires an API key.\n# You can get your own API key by signing up to OpenML.org)\nrun.publish()\nprint(f'View the run online: {run.openml_url}')\n</code></pre> <p>Find more examples in the sidebar on the left.</p>"},{"location":"python/#how-to-get-openml-for-python","title":"How to get OpenML for python","text":"<p>You can install the OpenML package via <code>pip</code> (we recommend using a virtual environment):</p> <pre><code>python -m pip install openml\n</code></pre> <p>For more advanced installation information, please see the \"Introduction\" example.</p>"},{"location":"python/#further-information","title":"Further information","text":"<ul> <li>OpenML documentation</li> <li>OpenML client APIs</li> <li>OpenML developer guide</li> <li>Contact information</li> <li>Citation request</li> <li>OpenML blog</li> <li>OpenML twitter account</li> </ul>"},{"location":"python/#contributing","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated. Please see the \"Contributing\" page for more information.</p>"},{"location":"python/#citing-openml-python","title":"Citing OpenML-Python","text":"<p>If you use OpenML-Python in a scientific publication, we would appreciate a reference to our JMLR-MLOSS paper  \"OpenML-Python: an extensible Python API for OpenML\":</p> BibtexMLA <pre><code>@article{JMLR:v22:19-920,\n    author  = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas M\u00c3\u00bcller and Joaquin Vanschoren and Frank Hutter},\n    title   = {OpenML-Python: an extensible Python API for OpenML},\n    journal = {Journal of Machine Learning Research},\n    year    = {2021},\n    volume  = {22},\n    number  = {100},\n    pages   = {1--5},\n    url     = {http://jmlr.org/papers/v22/19-920.html}\n}\n</code></pre> <p>Feurer, Matthias, et al.  \"OpenML-Python: an extensible Python API for OpenML.\" Journal of Machine Learning Research 22.100 (2021):1\u22125.</p>"},{"location":"python/CONTRIBUTING/","title":"CONTRIBUTING","text":"<p>This document describes the workflow on how to contribute to the openml-python package. If you are interested in connecting a machine learning package with OpenML (i.e. write an openml-python extension) or want to find other ways to contribute, see this page.</p>"},{"location":"python/CONTRIBUTING/#scope-of-the-package","title":"Scope of the package","text":"<p>The scope of the OpenML Python package is to provide a Python interface to the OpenML platform which integrates well with Python's scientific stack, most notably numpy, scipy and pandas. To reduce opportunity costs and demonstrate the usage of the package, it also implements an interface to the most popular machine learning package written in Python, scikit-learn. Thereby it will automatically be compatible with many machine learning libraries written in Python.</p> <p>We aim to keep the package as light-weight as possible and we will try to keep the number of potential installation dependencies as low as possible. Therefore, the connection to other machine learning libraries such as pytorch, keras or tensorflow should not be done directly inside this package, but in a separate package using the OpenML Python connector. More information on OpenML Python connectors can be found here.</p>"},{"location":"python/CONTRIBUTING/#reporting-bugs","title":"Reporting bugs","text":"<p>We use GitHub issues to track all bugs and feature requests; feel free to open an issue if you have found a bug or wish to see a feature implemented.</p> <p>It is recommended to check that your issue complies with the following rules before submitting:</p> <ul> <li> <p>Verify that your issue is not being currently addressed by other    issues    or pull requests.</p> </li> <li> <p>Please ensure all code snippets and error messages are formatted in    appropriate code blocks.    See Creating and highlighting code blocks.</p> </li> <li> <p>Please include your operating system type and version number, as well    as your Python, openml, scikit-learn, numpy, and scipy versions. This information    can be found by running the following code snippet: <pre><code>import platform; print(platform.platform())\nimport sys; print(\"Python\", sys.version)\nimport numpy; print(\"NumPy\", numpy.__version__)\nimport scipy; print(\"SciPy\", scipy.__version__)\nimport sklearn; print(\"Scikit-Learn\", sklearn.__version__)\nimport openml; print(\"OpenML\", openml.__version__)\n</code></pre></p> </li> </ul>"},{"location":"python/CONTRIBUTING/#determine-what-contribution-to-make","title":"Determine what contribution to make","text":"<p>Great! You've decided you want to help out. Now what? All contributions should be linked to issues on the Github issue tracker. In particular for new contributors, the good first issue label should help you find issues which are suitable for beginners.  Resolving these issues allow you to start contributing to the project without much prior knowledge. Your assistance in this area  will be greatly appreciated by the more experienced developers as it helps free up  their time to concentrate on other issues.</p> <p>If you encountered a particular part of the documentation or code that you want to improve, but there is no related open issue yet, open one first. This is important since you can first get feedback or pointers from experienced contributors.</p> <p>To let everyone know you are working on an issue, please leave a comment that states you will work on the issue (or, if you have the permission, assign yourself to the issue). This avoids double work!</p>"},{"location":"python/CONTRIBUTING/#general-git-workflow","title":"General git workflow","text":"<p>The preferred workflow for contributing to openml-python is to fork the main repository on GitHub, clone, check out the branch <code>develop</code>, and develop on a new branch branch. Steps:</p> <ol> <li> <p>Fork the project repository    by clicking on the 'Fork' button near the top right of the page. This creates    a copy of the code under your GitHub user account. For more details on    how to fork a repository see this guide.</p> </li> <li> <p>Clone your fork of the openml-python repo from your GitHub account to your local disk:</p> </li> </ol> <pre><code>$ git clone git@github.com:YourLogin/openml-python.git\n$ cd openml-python\n</code></pre> <ol> <li>Switch to the <code>develop</code> branch:</li> </ol> <pre><code>$ git checkout develop\n</code></pre> <ol> <li>Create a <code>feature</code> branch to hold your development changes:</li> </ol> <pre><code>$ git checkout -b feature/my-feature\n</code></pre> <p>Always use a <code>feature</code> branch. It's good practice to never work on the <code>main</code> or <code>develop</code> branch!     To make the nature of your pull request easily visible, please prepend the name of the branch with the type of changes you want to merge, such as <code>feature</code> if it contains a new feature, <code>fix</code> for a bugfix, <code>doc</code> for documentation and <code>maint</code> for other maintenance on the package.</p> <ol> <li>Develop the feature on your feature branch. Add changed files using <code>git add</code> and then <code>git commit</code> files:</li> </ol> <pre><code>$ git add modified_files\n$ git commit\n</code></pre> <p>to record your changes in Git, then push the changes to your GitHub account with:</p> <pre><code>$ git push -u origin my-feature\n</code></pre> <ol> <li>Follow these instructions to create a pull request from your fork. This will send an email to the committers.</li> </ol> <p>(If any of the above seems like magic to you, please look up the Git documentation on the web, or ask a friend or another contributor for help.)</p>"},{"location":"python/CONTRIBUTING/#pull-request-checklist","title":"Pull Request Checklist","text":"<p>We recommended that your contribution complies with the following rules before you submit a pull request:</p> <ul> <li> <p>Follow the    pep8 style guide.    With the following exceptions or additions:</p> <ul> <li>The max line length is 100 characters instead of 80.</li> <li>When creating a multi-line expression with binary operators, break before the operator.</li> <li>Add type hints to all function signatures. (note: not all functions have type hints yet, this is work in progress.)</li> <li>Use the <code>str.format</code> over <code>printf</code> style formatting.  E.g. use <code>\"{} {}\".format('hello', 'world')</code> not <code>\"%s %s\" % ('hello', 'world')</code>.  (note: old code may still use <code>printf</code>-formatting, this is work in progress.)</li> </ul> </li> <li> <p>If your pull request addresses an issue, please use the pull request title    to describe the issue and mention the issue number in the pull request description. This will make sure a link back to the original issue is    created.</p> </li> <li> <p>An incomplete contribution -- where you expect to do more work before    receiving a full review -- should be submitted as a <code>draft</code>. These may be useful    to: indicate you are working on something to avoid duplicated work,    request broad review of functionality or API, or seek collaborators.    Drafts often benefit from the inclusion of a    task list    in the PR description.</p> </li> <li> <p>Add unit tests and examples for any new functionality being introduced. </p> <ul> <li>If an unit test contains an upload to the test server, please ensure that it is followed by a file collection for deletion, to prevent the test server from bulking up. For example, <code>TestBase._mark_entity_for_removal('data', dataset.dataset_id)</code>, <code>TestBase._mark_entity_for_removal('flow', (flow.flow_id, flow.name))</code>.</li> <li>Please ensure that the example is run on the test server by beginning with the call to <code>openml.config.start_using_configuration_for_example()</code>.</li> <li>Add the <code>@pytest.mark.sklearn</code> marker to your unit tests if they have a dependency on scikit-learn.</li> </ul> </li> <li> <p>All tests pass when running <code>pytest</code>. On    Unix-like systems, check with (from the toplevel source folder):</p> <pre><code>$ pytest\n</code></pre> </li> </ul> <p>For Windows systems, execute the command from an Anaconda Prompt or add <code>pytest</code> to PATH before executing the command.</p> <ul> <li> <p>Documentation and high-coverage tests are necessary for enhancements to be    accepted. Bug-fixes or new features should be provided with    non-regression tests.    These tests verify the correct behavior of the fix or feature. In this    manner, further modifications on the code base are granted to be consistent    with the desired behavior.    For the Bug-fixes case, at the time of the PR, this tests should fail for    the code base in develop and pass for the PR code.</p> </li> <li> <p>Add your changes to the changelog in the file doc/progress.rst.</p> </li> <li> <p>If any source file is being added to the repository, please add the BSD 3-Clause license to it.</p> </li> </ul> <p>Note: We recommend to follow the instructions below to install all requirements locally. However it is also possible to use the openml-python docker image for testing and building documentation. This can be useful for one-off contributions or when you are experiencing installation issues.</p> <p>First install openml with its test dependencies by running   <pre><code>$ pip install -e .[test]\n</code></pre> from the repository folder. Then configure pre-commit through  <pre><code>$ pre-commit install\n</code></pre> This will install dependencies to run unit tests, as well as pre-commit. To run the unit tests, and check their code coverage, run:   <pre><code>$ pytest --cov=. path/to/tests_for_package\n</code></pre> Make sure your code has good unittest coverage (at least 80%).</p> <p>Pre-commit is used for various style checking and code formatting. Before each commit, it will automatically run:  - black a code formatter.    This will automatically format your code.    Make sure to take a second look after any formatting takes place,    if the resulting code is very bloated, consider a (small) refactor.    note: If Black reformats your code, the commit will automatically be aborted.    Make sure to add the formatted files (back) to your commit after checking them.  - mypy a static type checker.    In particular, make sure each function you work on has type hints.  - flake8 style guide enforcement.    Almost all of the black-formatted code should automatically pass this check,    but make sure to make adjustments if it does fail.</p> <p>If you want to run the pre-commit tests without doing a commit, run: <pre><code>$ make check\n</code></pre> or on a system without make, like Windows: <pre><code>$ pre-commit run --all-files\n</code></pre> Make sure to do this at least once before your first commit to check your setup works.</p> <p>Executing a specific unit test can be done by specifying the module, test case, and test. To obtain a hierarchical list of all tests, run</p> <pre><code>$  pytest --collect-only\n\n &lt;Module 'tests/test_datasets/test_dataset.py'&gt;\n   &lt;UnitTestCase 'OpenMLDatasetTest'&gt;\n     &lt;TestCaseFunction 'test_dataset_format_constructor'&gt;\n     &lt;TestCaseFunction 'test_get_data'&gt;\n     &lt;TestCaseFunction 'test_get_data_rowid_and_ignore_and_target'&gt;\n     &lt;TestCaseFunction 'test_get_data_with_ignore_attributes'&gt;\n     &lt;TestCaseFunction 'test_get_data_with_rowid'&gt;\n     &lt;TestCaseFunction 'test_get_data_with_target'&gt;\n   &lt;UnitTestCase 'OpenMLDatasetTestOnTestServer'&gt;\n     &lt;TestCaseFunction 'test_tagging'&gt;\n</code></pre> <p>You may then run a specific module, test case, or unit test respectively: <pre><code>  $ pytest tests/test_datasets/test_dataset.py\n  $ pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest\n  $ pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest::test_get_data\n</code></pre></p> <p>NOTE: In the case the examples build fails during the Continuous Integration test online, please  fix the first failing example. If the first failing example switched the server from live to test  or vice-versa, and the subsequent examples expect the other server, the ensuing examples will fail  to be built as well.</p> <p>Happy testing!</p>"},{"location":"python/CONTRIBUTING/#documentation","title":"Documentation","text":"<p>We are glad to accept any sort of documentation: function docstrings, reStructuredText documents, tutorials, etc. reStructuredText documents live in the source code repository under the doc/ directory.</p> <p>You can edit the documentation using any text editor and then generate the HTML output by typing <code>make html</code> from the doc/ directory. The resulting HTML files will be placed in <code>build/html/</code> and are viewable in a web browser. See the <code>README</code> file in the <code>doc/</code> directory for more information.</p> <p>For building the documentation, you will need to install a few additional dependencies: <pre><code>$ pip install -e .[docs]\n</code></pre> When dependencies are installed, run <pre><code>$ sphinx-build -b html doc YOUR_PREFERRED_OUTPUT_DIRECTORY\n</code></pre></p>"},{"location":"python/ISSUE_TEMPLATE/","title":"ISSUE TEMPLATE","text":""},{"location":"python/ISSUE_TEMPLATE/#description","title":"Description","text":""},{"location":"python/ISSUE_TEMPLATE/#stepscode-to-reproduce","title":"Steps/Code to Reproduce","text":""},{"location":"python/ISSUE_TEMPLATE/#expected-results","title":"Expected Results","text":""},{"location":"python/ISSUE_TEMPLATE/#actual-results","title":"Actual Results","text":""},{"location":"python/ISSUE_TEMPLATE/#versions","title":"Versions","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/","title":"PULL REQUEST TEMPLATE","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#reference-issue","title":"Reference Issue","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#what-does-this-pr-implementfix-explain-your-changes","title":"What does this PR implement/fix? Explain your changes.","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#how-should-this-pr-be-tested","title":"How should this PR be tested?","text":""},{"location":"python/PULL_REQUEST_TEMPLATE/#any-other-comments","title":"Any other comments?","text":""},{"location":"python/contributing/","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated in all forms. In particular, a few ways to contribute to openml-python are:</p> <ul> <li>A direct contribution to the package, by means of improving the     code, documentation or examples. To get started, see this     file     with details on how to set up your environment to develop for     openml-python.</li> <li>A contribution to an openml-python extension. An extension package     allows OpenML to interface with a machine learning package (such     as scikit-learn or keras). These extensions are hosted in separate     repositories and may have their own guidelines. For more     information, see also extensions.</li> <li>Bug reports. If something doesn't work for you or is cumbersome,     please open a new issue to let us know about the problem. See     this     section.</li> <li>Cite OpenML if you use it in a     scientific publication.</li> <li>Visit one of our hackathons.</li> <li>Contribute to another OpenML project, such as the main OpenML     project.</li> </ul>"},{"location":"python/extensions/","title":"Extensions","text":"<p>OpenML-Python provides an extension interface to connect other machine learning libraries than scikit-learn to OpenML. Please check the <code>api_extensions</code> and use the scikit-learn extension in <code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text role=\"class\"} as a starting point.</p>"},{"location":"python/extensions/#list-of-extensions","title":"List of extensions","text":"<p>Here is a list of currently maintained OpenML extensions:</p> <ul> <li><code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text     role=\"class\"}</li> <li>openml-keras</li> <li>openml-pytorch</li> <li>openml-tensorflow (for tensorflow     2+)</li> </ul>"},{"location":"python/extensions/#connecting-new-machine-learning-libraries","title":"Connecting new machine learning libraries","text":""},{"location":"python/extensions/#content-of-the-library","title":"Content of the Library","text":"<p>To leverage support from the community and to tap in the potential of OpenML, interfacing with popular machine learning libraries is essential. The OpenML-Python package is capable of downloading meta-data and results (data, flows, runs), regardless of the library that was used to upload it. However, in order to simplify the process of uploading flows and runs from a specific library, an additional interface can be built. The OpenML-Python team does not have the capacity to develop and maintain such interfaces on its own. For this reason, we have built an extension interface to allows others to contribute back. Building a suitable extension for therefore requires an understanding of the current OpenML-Python support.</p> <p>The <code>sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py</code>{.interpreted-text role=\"ref\"} tutorial shows how scikit-learn currently works with OpenML-Python as an extension. The sklearn extension packaged with the openml-python repository can be used as a template/benchmark to build the new extension.</p>"},{"location":"python/extensions/#api","title":"API","text":"<ul> <li>The extension scripts must import the [openml]{.title-ref} package     and be able to interface with any function from the OpenML-Python     <code>api</code>.</li> <li>The extension has to be defined as a Python class and must inherit     from <code>openml.extensions.Extension</code>.</li> <li>This class needs to have all the functions from [class     Extension]{.title-ref} overloaded as required.</li> <li>The redefined functions should have adequate and appropriate     docstrings. The [Sklearn Extension API     :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref}     is a good example to follow.</li> </ul>"},{"location":"python/extensions/#interfacing-with-openml-python","title":"Interfacing with OpenML-Python","text":"<p>Once the new extension class has been defined, the openml-python module to <code>openml.extensions.register_extension</code> must be called to allow OpenML-Python to interface the new extension.</p> <p>The following methods should get implemented. Although the documentation in the [Extension]{.title-ref} interface should always be leading, here we list some additional information and best practices. The [Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is a good example to follow. Note that most methods are relatively simple and can be implemented in several lines of code.</p> <ul> <li>General setup (required)<ul> <li><code>can_handle_flow</code>: Takes as     argument an OpenML flow, and checks whether this can be handled     by the current extension. The OpenML database consists of many     flows, from various workbenches (e.g., scikit-learn, Weka, mlr).     This method is called before a model is being deserialized.     Typically, the flow-dependency field is used to check whether     the specific library is present, and no unknown libraries are     present there.</li> <li><code>can_handle_model</code>: Similar as     <code>can_handle_flow</code>, except that in     this case a Python object is given. As such, in many cases, this     method can be implemented by checking whether this adheres to a     certain base class.</li> </ul> </li> <li>Serialization and De-serialization (required)<ul> <li><code>flow_to_model</code>: deserializes the     OpenML Flow into a model (if the library can indeed handle the     flow). This method has an important interplay with     <code>model_to_flow</code>. Running these     two methods in succession should result in exactly the same     model (or flow). This property can be used for unit testing     (e.g., build a model with hyperparameters, make predictions on a     task, serialize it to a flow, deserialize it back, make it     predict on the same task, and check whether the predictions are     exactly the same.) The example in the scikit-learn interface     might seem daunting, but note that here some complicated design     choices were made, that allow for all sorts of interesting     research questions. It is probably good practice to start easy.</li> <li><code>model_to_flow</code>: The inverse of     <code>flow_to_model</code>. Serializes a     model into an OpenML Flow. The flow should preserve the class,     the library version, and the tunable hyperparameters.</li> <li><code>get_version_information</code>: Return     a tuple with the version information of the important libraries.</li> <li><code>create_setup_string</code>: No longer     used, and will be deprecated soon.</li> </ul> </li> <li>Performing runs (required)<ul> <li><code>is_estimator</code>: Gets as input a     class, and checks whether it has the status of estimator in the     library (typically, whether it has a train method and a predict     method).</li> <li><code>seed_model</code>: Sets a random seed     to the model.</li> <li><code>_run_model_on_fold</code>: One of the     main requirements for a library to generate run objects for the     OpenML server. Obtains a train split (with labels) and a test     split (without labels) and the goal is to train a model on the     train split and return the predictions on the test split. On top     of the actual predictions, also the class probabilities should     be determined. For classifiers that do not return class     probabilities, this can just be the hot-encoded predicted label.     The predictions will be evaluated on the OpenML server. Also,     additional information can be returned, for example,     user-defined measures (such as runtime information, as this can     not be inferred on the server). Additionally, information about     a hyperparameter optimization trace can be provided.</li> <li><code>obtain_parameter_values</code>:     Obtains the hyperparameters of a given model and the current     values. Please note that in the case of a hyperparameter     optimization procedure (e.g., random search), you only should     return the hyperparameters of this procedure (e.g., the     hyperparameter grid, budget, etc) and that the chosen model will     be inferred from the optimization trace.</li> <li><code>check_if_model_fitted</code>: Check     whether the train method of the model has been called (and as     such, whether the predict method can be used).</li> </ul> </li> <li>Hyperparameter optimization (optional)<ul> <li><code>instantiate_model_from_hpo_class</code>{.interpreted-text     role=\"meth\"}: If a given run has recorded the hyperparameter     optimization trace, then this method can be used to     reinstantiate the model with hyperparameters of a given     hyperparameter optimization iteration. Has some similarities     with <code>flow_to_model</code> (as this     method also sets the hyperparameters of a model). Note that     although this method is required, it is not necessary to     implement any logic if hyperparameter optimization is not     implemented. Simply raise a [NotImplementedError]{.title-ref}     then.</li> </ul> </li> </ul>"},{"location":"python/extensions/#hosting-the-library","title":"Hosting the library","text":"<p>Each extension created should be a stand-alone repository, compatible with the OpenML-Python repository. The extension repository should work off-the-shelf with OpenML-Python installed.</p> <p>Create a public Github repo with the following directory structure:</p> <pre><code>| [repo name]\n|    |-- [extension name]\n|    |    |-- __init__.py\n|    |    |-- extension.py\n|    |    |-- config.py (optionally)\n</code></pre>"},{"location":"python/extensions/#recommended","title":"Recommended","text":"<ul> <li>Test cases to keep the extension up to date with the     [openml-python]{.title-ref} upstream changes.</li> <li>Documentation of the extension API, especially if any new     functionality added to OpenML-Python\\'s extension design.</li> <li>Examples to show how the new extension interfaces and works with     OpenML-Python.</li> <li>Create a PR to add the new extension to the OpenML-Python API     documentation.</li> </ul> <p>Happy contributing!</p>"},{"location":"python/progress/","title":"Changelog","text":""},{"location":"python/progress/#next","title":"next","text":"<ul> <li>MAINT #1340: Add Numpy 2.0 support. Update tests to work with     scikit-learn \\&lt;= 1.5.</li> <li>ADD #1342: Add HTTP header to requests to indicate they are from     openml-python.</li> </ul>"},{"location":"python/progress/#0142","title":"0.14.2","text":"<ul> <li>MAINT #1280: Use the server-provided <code>parquet_url</code> instead of     <code>minio_url</code> to determine the location of the parquet file.</li> <li>ADD #716: add documentation for remaining attributes of classes     and functions.</li> <li>ADD #1261: more annotations for type hints.</li> <li>MAINT #1294: update tests to new tag specification.</li> <li>FIX #1314: Update fetching a bucket from MinIO.</li> <li>FIX #1315: Make class label retrieval more lenient.</li> <li>ADD #1316: add feature descriptions ontologies support.</li> <li>MAINT #1310/#1307: switch to ruff and resolve all mypy errors.</li> </ul>"},{"location":"python/progress/#0141","title":"0.14.1","text":"<ul> <li>FIX: Fallback on downloading ARFF when failing to download parquet     from MinIO due to a ServerError.</li> </ul>"},{"location":"python/progress/#0140","title":"0.14.0","text":"<p>IMPORTANT: This release paves the way towards a breaking update of OpenML-Python. From version 0.15, functions that had the option to return a pandas DataFrame will return a pandas DataFrame by default. This version (0.14) emits a warning if you still use the old access functionality. More concretely:</p> <ul> <li>In 0.15 we will drop the ability to return dictionaries in listing     calls and only provide pandas DataFrames. To disable warnings in     0.14 you have to request a pandas DataFrame (using     <code>output_format=\"dataframe\"</code>).</li> <li>In 0.15 we will drop the ability to return datasets as numpy arrays     and only provide pandas DataFrames. To disable warnings in 0.14 you     have to request a pandas DataFrame (using     <code>dataset_format=\"dataframe\"</code>).</li> </ul> <p>Furthermore, from version 0.15, OpenML-Python will no longer download datasets and dataset metadata by default. This version (0.14) emits a warning if you don\\'t explicitly specifiy the desired behavior.</p> <p>Please see the pull requests #1258 and #1260 for further information.</p> <ul> <li>ADD #1081: New flag that allows disabling downloading dataset     features.</li> <li>ADD #1132: New flag that forces a redownload of cached data.</li> <li>FIX #1244: Fixes a rare bug where task listing could fail when the     server returned invalid data.</li> <li>DOC #1229: Fixes a comment string for the main example.</li> <li>DOC #1241: Fixes a comment in an example.</li> <li>MAINT #1124: Improve naming of helper functions that govern the     cache directories.</li> <li>MAINT #1223, #1250: Update tools used in pre-commit to the latest     versions (<code>black==23.30</code>, <code>mypy==1.3.0</code>, <code>flake8==6.0.0</code>).</li> <li>MAINT #1253: Update the citation request to the JMLR paper.</li> <li>MAINT #1246: Add a warning that warns the user that checking for     duplicate runs on the server cannot be done without an API key.</li> </ul>"},{"location":"python/progress/#0131","title":"0.13.1","text":"<ul> <li>ADD #1081 #1132: Add additional options for (not) downloading     datasets <code>openml.datasets.get_dataset</code> and cache management.</li> <li>ADD #1028: Add functions to delete runs, flows, datasets, and tasks     (e.g., <code>openml.datasets.delete_dataset</code>).</li> <li>ADD #1144: Add locally computed results to the <code>OpenMLRun</code> object\\'s     representation if the run was created locally and not downloaded     from the server.</li> <li>ADD #1180: Improve the error message when the checksum of a     downloaded dataset does not match the checksum provided by the API.</li> <li>ADD #1201: Make <code>OpenMLTraceIteration</code> a dataclass.</li> <li>DOC #1069: Add argument documentation for the <code>OpenMLRun</code> class.</li> <li>DOC #1241 #1229 #1231: Minor documentation fixes and resolve     documentation examples not working.</li> <li>FIX #1197 #559 #1131: Fix the order of ground truth and predictions     in the <code>OpenMLRun</code> object and in <code>format_prediction</code>.</li> <li>FIX #1198: Support numpy 1.24 and higher.</li> <li>FIX #1216: Allow unknown task types on the server. This is only     relevant when new task types are added to the test server.</li> <li>FIX #1223: Fix mypy errors for implicit optional typing.</li> <li>MAINT #1155: Add dependabot github action to automatically update     other github actions.</li> <li>MAINT #1199: Obtain pre-commit\\'s flake8 from github.com instead of     gitlab.com.</li> <li>MAINT #1215: Support latest numpy version.</li> <li>MAINT #1218: Test Python3.6 on Ubuntu 20.04 instead of the latest     Ubuntu (which is 22.04).</li> <li>MAINT #1221 #1212 #1206 #1211: Update github actions to the latest     versions.</li> </ul>"},{"location":"python/progress/#0130","title":"0.13.0","text":"<ul> <li>FIX #1030: <code>pre-commit</code> hooks now no longer should issue a     warning.</li> <li>FIX #1058, #1100: Avoid <code>NoneType</code> error when printing task     without <code>class_labels</code> attribute.</li> <li>FIX #1110: Make arguments to <code>create_study</code> and <code>create_suite</code>     that are defined as optional by the OpenML XSD actually optional.</li> <li>FIX #1147: <code>openml.flow.flow_exists</code> no longer requires an API     key.</li> <li>FIX #1184: Automatically resolve proxies when downloading from     minio. Turn this off by setting environment variable     <code>no_proxy=\"*\"</code>.</li> <li>MAINT #1088: Do CI for Windows on Github Actions instead of     Appveyor.</li> <li>MAINT #1104: Fix outdated docstring for <code>list_task</code>.</li> <li>MAINT #1146: Update the pre-commit dependencies.</li> <li>ADD #1103: Add a <code>predictions</code> property to OpenMLRun for easy     accessibility of prediction data.</li> <li>ADD #1188: EXPERIMENTAL. Allow downloading all files from a minio     bucket with <code>download_all_files=True</code> for <code>get_dataset</code>.</li> </ul>"},{"location":"python/progress/#0122","title":"0.12.2","text":"<ul> <li>ADD #1065: Add a <code>retry_policy</code> configuration option that determines     the frequency and number of times to attempt to retry server     requests.</li> <li>ADD #1075: A docker image is now automatically built on a push to     develop. It can be used to build docs or run tests in an isolated     environment.</li> <li>ADD: You can now avoid downloading \\'qualities\\' meta-data when     downloading a task with the <code>download_qualities</code> parameter of     <code>openml.tasks.get_task[s]</code> functions.</li> <li>DOC: Fixes a few broken links in the documentation.</li> <li>DOC #1061: Improve examples to always show a warning when they     switch to the test server.</li> <li>DOC #1067: Improve documentation on the scikit-learn extension     interface.</li> <li>DOC #1068: Create dedicated extensions page.</li> <li>FIX #1075: Correctly convert [y]{.title-ref} to a pandas series when     downloading sparse data.</li> <li>MAINT: Rename [master]{.title-ref} brach to [ main]{.title-ref}     branch.</li> <li>MAINT/DOC: Automatically check for broken external links when     building the documentation.</li> <li>MAINT/DOC: Fail documentation building on warnings. This will make     the documentation building fail if a reference cannot be found (i.e.     an internal link is broken).</li> </ul>"},{"location":"python/progress/#0121","title":"0.12.1","text":"<ul> <li>ADD #895/#1038: Measure runtimes of scikit-learn runs also for     models which are parallelized via the joblib.</li> <li>DOC #1050: Refer to the webpage instead of the XML file in the main     example.</li> <li>DOC #1051: Document existing extensions to OpenML-Python besides the     shipped scikit-learn extension.</li> <li>FIX #1035: Render class attributes and methods again.</li> <li>ADD #1049: Add a command line tool for configuration openml-python.</li> <li>FIX #1042: Fixes a rare concurrency issue with OpenML-Python and     joblib which caused the joblib worker pool to fail.</li> <li>FIX #1053: Fixes a bug which could prevent importing the package in     a docker container.</li> </ul>"},{"location":"python/progress/#0120","title":"0.12.0","text":"<ul> <li>ADD #964: Validate <code>ignore_attribute</code>, <code>default_target_attribute</code>,     <code>row_id_attribute</code> are set to attributes that exist on the dataset     when calling <code>create_dataset</code>.</li> <li>ADD #979: Dataset features and qualities are now also cached in     pickle format.</li> <li>ADD #982: Add helper functions for column transformers.</li> <li>ADD #989: <code>run_model_on_task</code> will now warn the user the the model     passed has already been fitted.</li> <li>ADD #1009 : Give possibility to not download the dataset qualities.     The cached version is used even so download attribute is false.</li> <li>ADD #1016: Add scikit-learn 0.24 support.</li> <li>ADD #1020: Add option to parallelize evaluation of tasks with     joblib.</li> <li>ADD #1022: Allow minimum version of dependencies to be listed for a     flow, use more accurate minimum versions for scikit-learn     dependencies.</li> <li>ADD #1023: Add admin-only calls for adding topics to datasets.</li> <li>ADD #1029: Add support for fetching dataset from a minio server in     parquet format.</li> <li>ADD #1031: Generally improve runtime measurements, add them for some     previously unsupported flows (e.g. BaseSearchCV derived flows).</li> <li>DOC #973 : Change the task used in the welcome page example so it no     longer fails using numerical dataset.</li> <li>MAINT #671: Improved the performance of <code>check_datasets_active</code> by     only querying the given list of datasets in contrast to querying all     datasets. Modified the corresponding unit test.</li> <li>MAINT #891: Changed the way that numerical features are stored.     Numerical features that range from 0 to 255 are now stored as uint8,     which reduces the storage space required as well as storing and     loading times.</li> <li>MAINT #975, #988: Add CI through Github Actions.</li> <li>MAINT #977: Allow <code>short</code> and <code>long</code> scenarios for unit tests.     Reduce the workload for some unit tests.</li> <li>MAINT #985, #1000: Improve unit test stability and output     readability, and adds load balancing.</li> <li>MAINT #1018: Refactor data loading and storage. Data is now     compressed on the first call to [get_data]{.title-ref}.</li> <li>MAINT #1024: Remove flaky decorator for study unit test.</li> <li>FIX #883 #884 #906 #972: Various improvements to the caching system.</li> <li>FIX #980: Speed up <code>check_datasets_active</code>.</li> <li>FIX #984: Add a retry mechanism when the server encounters a     database issue.</li> <li>FIX #1004: Fixed an issue that prevented installation on some     systems (e.g. Ubuntu).</li> <li>FIX #1013: Fixes a bug where <code>OpenMLRun.setup_string</code> was not     uploaded to the server, prepares for <code>run_details</code> being sent from     the server.</li> <li>FIX #1021: Fixes an issue that could occur when running unit tests     and openml-python was not in PATH.</li> <li>FIX #1037: Fixes a bug where a dataset could not be loaded if a     categorical value had listed nan-like as a possible category.</li> </ul>"},{"location":"python/progress/#0110","title":"0.11.0","text":"<ul> <li>ADD #753: Allows uploading custom flows to OpenML via OpenML-Python.</li> <li>ADD #777: Allows running a flow on pandas dataframes (in addition to     numpy arrays).</li> <li>ADD #888: Allow passing a [task_id]{.title-ref} to     [run_model_on_task]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #929: Add <code>edit_dataset</code> and <code>fork_dataset</code> to allow editing and     forking of uploaded datasets.</li> <li>ADD #866, #943: Add support for scikit-learn\\'s     [passthrough]{.title-ref} and [drop]{.title-ref} when uploading     flows to OpenML.</li> <li>ADD #879: Add support for scikit-learn\\'s MLP hyperparameter     [layer_sizes]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #945: PEP 561 compliance for distributing Type information.</li> <li>DOC #660: Remove nonexistent argument from docstring.</li> <li>DOC #901: The API reference now documents the config file and its     options.</li> <li>DOC #912: API reference now shows [create_task]{.title-ref}.</li> <li>DOC #954: Remove TODO text from documentation.</li> <li>DOC #960: document how to upload multiple ignore attributes.</li> <li>FIX #873: Fixes an issue which resulted in incorrect URLs when     printing OpenML objects after switching the server.</li> <li>FIX #885: Logger no longer registered by default. Added utility     functions to easily register logging to console and file.</li> <li>FIX #890: Correct the scaling of data in the SVM example.</li> <li>MAINT #371: <code>list_evaluations</code> default <code>size</code> changed from <code>None</code> to     <code>10_000</code>.</li> <li>MAINT #767: Source distribution installation is now unit-tested.</li> <li>MAINT #781: Add pre-commit and automated code formatting with black.</li> <li>MAINT #804: Rename arguments of list_evaluations to indicate they     expect lists of ids.</li> <li>MAINT #836: OpenML supports only pandas version 1.0.0 or above.</li> <li>MAINT #865: OpenML no longer bundles test files in the source     distribution.</li> <li>MAINT #881: Improve the error message for too-long URIs.</li> <li>MAINT #897: Dropping support for Python 3.5.</li> <li>MAINT #916: Adding support for Python 3.8.</li> <li>MAINT #920: Improve error messages for dataset upload.</li> <li>MAINT #921: Improve hangling of the OpenML server URL in the config     file.</li> <li>MAINT #925: Improve error handling and error message when loading     datasets.</li> <li>MAINT #928: Restructures the contributing documentation.</li> <li>MAINT #936: Adding support for scikit-learn 0.23.X.</li> <li>MAINT #945: Make OpenML-Python PEP562 compliant.</li> <li>MAINT #951: Converts TaskType class to a TaskType enum.</li> </ul>"},{"location":"python/progress/#0102","title":"0.10.2","text":"<ul> <li>ADD #857: Adds task type ID to list_runs</li> <li>DOC #862: Added license BSD 3-Clause to each of the source files.</li> </ul>"},{"location":"python/progress/#0101","title":"0.10.1","text":"<ul> <li>ADD #175: Automatically adds the docstring of scikit-learn objects     to flow and its parameters.</li> <li>ADD #737: New evaluation listing call that includes the     hyperparameter settings.</li> <li>ADD #744: It is now possible to only issue a warning and not raise     an exception if the package versions for a flow are not met when     deserializing it.</li> <li>ADD #783: The URL to download the predictions for a run is now     stored in the run object.</li> <li>ADD #790: Adds the uploader name and id as new filtering options for     <code>list_evaluations</code>.</li> <li>ADD #792: New convenience function <code>openml.flow.get_flow_id</code>.</li> <li>ADD #861: Debug-level log information now being written to a file in     the cache directory (at most 2 MB).</li> <li>DOC #778: Introduces instructions on how to publish an extension to     support other libraries than scikit-learn.</li> <li>DOC #785: The examples section is completely restructured into     simple simple examples, advanced examples and examples showcasing     the use of OpenML-Python to reproduce papers which were done with     OpenML-Python.</li> <li>DOC #788: New example on manually iterating through the split of a     task.</li> <li>DOC #789: Improve the usage of dataframes in the examples.</li> <li>DOC #791: New example for the paper Efficient and Robust Automated     Machine Learning by Feurer et al. (2015).</li> <li>DOC #803: New example for the paper Don't Rule Out Simple Models     Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear     Classifiers in OpenML by Benjamin Strang et al. (2018).</li> <li>DOC #808: New example demonstrating basic use cases of a dataset.</li> <li>DOC #810: New example demonstrating the use of benchmarking studies     and suites.</li> <li>DOC #832: New example for the paper Scalable Hyperparameter     Transfer Learning by Valerio Perrone et al. (2019)</li> <li>DOC #834: New example showing how to plot the loss surface for a     support vector machine.</li> <li>FIX #305: Do not require the external version in the flow XML when     loading an object.</li> <li>FIX #734: Better handling of \\\"old\\\" flows.</li> <li>FIX #736: Attach a StreamHandler to the openml logger instead of the     root logger.</li> <li>FIX #758: Fixes an error which made the client API crash when     loading a sparse data with categorical variables.</li> <li>FIX #779: Do not fail on corrupt pickle</li> <li>FIX #782: Assign the study id to the correct class attribute.</li> <li>FIX #819: Automatically convert column names to type string when     uploading a dataset.</li> <li>FIX #820: Make <code>__repr__</code> work for datasets which do not have an id.</li> <li>MAINT #796: Rename an argument to make the function     <code>list_evaluations</code> more consistent.</li> <li>MAINT #811: Print the full error message given by the server.</li> <li>MAINT #828: Create base class for OpenML entity classes.</li> <li>MAINT #829: Reduce the number of data conversion warnings.</li> <li>MAINT #831: Warn if there\\'s an empty flow description when     publishing a flow.</li> <li>MAINT #837: Also print the flow XML if a flow fails to validate.</li> <li>FIX #838: Fix list_evaluations_setups to work when evaluations are     not a 100 multiple.</li> <li>FIX #847: Fixes an issue where the client API would crash when     trying to download a dataset when there are no qualities available     on the server.</li> <li>MAINT #849: Move logic of most different <code>publish</code> functions into     the base class.</li> <li>MAINt #850: Remove outdated test code.</li> </ul>"},{"location":"python/progress/#0100","title":"0.10.0","text":"<ul> <li>ADD #737: Add list_evaluations_setups to return hyperparameters     along with list of evaluations.</li> <li>FIX #261: Test server is cleared of all files uploaded during unit     testing.</li> <li>FIX #447: All files created by unit tests no longer persist in     local.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>FIX #447: All files created by unit tests are deleted after the     completion of all unit tests.</li> <li>FIX #589: Fixing a bug that did not successfully upload the columns     to ignore when creating and publishing a dataset.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>DOC #639: More descriptive documention for function to convert array     format.</li> <li>DOC #719: Add documentation on uploading tasks.</li> <li>ADD #687: Adds a function to retrieve the list of evaluation     measures available.</li> <li>ADD #695: A function to retrieve all the data quality measures     available.</li> <li>ADD #412: Add a function to trim flow names for scikit-learn flows.</li> <li>ADD #715: [list_evaluations]{.title-ref} now has an option to sort     evaluations by score (value).</li> <li>ADD #722: Automatic reinstantiation of flow in     [run_model_on_task]{.title-ref}. Clearer errors if that\\'s not     possible.</li> <li>ADD #412: The scikit-learn extension populates the short name field     for flows.</li> <li>MAINT #726: Update examples to remove deprecation warnings from     scikit-learn</li> <li>MAINT #752: Update OpenML-Python to be compatible with sklearn 0.21</li> <li>ADD #790: Add user ID and name to list_evaluations</li> </ul>"},{"location":"python/progress/#090","title":"0.9.0","text":"<ul> <li>ADD #560: OpenML-Python can now handle regression tasks as well.</li> <li>ADD #620, #628, #632, #649, #682: Full support for studies and     distinguishes suites from studies.</li> <li>ADD #607: Tasks can now be created and uploaded.</li> <li>ADD #647, #673: Introduced the extension interface. This provides an     easy way to create a hook for machine learning packages to perform     e.g. automated runs.</li> <li>ADD #548, #646, #676: Support for Pandas DataFrame and     SparseDataFrame</li> <li>ADD #662: Results of listing functions can now be returned as     pandas.DataFrame.</li> <li>ADD #59: Datasets can now also be retrieved by name.</li> <li>ADD #672: Add timing measurements for runs, when possible.</li> <li>ADD #661: Upload time and error messages now displayed with     [list_runs]{.title-ref}.</li> <li>ADD #644: Datasets can now be downloaded \\'lazily\\', retrieving only     metadata at first, and the full dataset only when necessary.</li> <li>ADD #659: Lazy loading of task splits.</li> <li>ADD #516: [run_flow_on_task]{.title-ref} flow uploading is now     optional.</li> <li>ADD #680: Adds     [openml.config.start_using_configuration_for_example]{.title-ref}     (and resp. stop) to easily connect to the test server.</li> <li>ADD #75, #653: Adds a pretty print for objects of the top-level     classes.</li> <li>FIX #642: [check_datasets_active]{.title-ref} now correctly also     returns active status of deactivated datasets.</li> <li>FIX #304, #636: Allow serialization of numpy datatypes and list of     lists of more types (e.g. bools, ints) for flows.</li> <li>FIX #651: Fixed a bug that would prevent openml-python from finding     the user\\'s config file.</li> <li>FIX #693: OpenML-Python uses liac-arff instead of scipy.io for     loading task splits now.</li> <li>DOC #678: Better color scheme for code examples in documentation.</li> <li>DOC #681: Small improvements and removing list of missing functions.</li> <li>DOC #684: Add notice to examples that connect to the test server.</li> <li>DOC #688: Add new example on retrieving evaluations.</li> <li>DOC #691: Update contributing guidelines to use Github draft feature     instead of tags in title.</li> <li>DOC #692: All functions are documented now.</li> <li>MAINT #184: Dropping Python2 support.</li> <li>MAINT #596: Fewer dependencies for regular pip install.</li> <li>MAINT #652: Numpy and Scipy are no longer required before     installation.</li> <li>MAINT #655: Lazy loading is now preferred in unit tests.</li> <li>MAINT #667: Different tag functions now share code.</li> <li>MAINT #666: More descriptive error message for     [TypeError]{.title-ref} in [list_runs]{.title-ref}.</li> <li>MAINT #668: Fix some type hints.</li> <li>MAINT #677: [dataset.get_data]{.title-ref} now has consistent     behavior in its return type.</li> <li>MAINT #686: Adds ignore directives for several [mypy]{.title-ref}     folders.</li> <li>MAINT #629, #630: Code now adheres to single PEP8 standard.</li> </ul>"},{"location":"python/progress/#080","title":"0.8.0","text":"<ul> <li>ADD #440: Improved dataset upload.</li> <li>ADD #545, #583: Allow uploading a dataset from a pandas DataFrame.</li> <li>ADD #528: New functions to update the status of a dataset.</li> <li>ADD #523: Support for scikit-learn 0.20\\'s new ColumnTransformer.</li> <li>ADD #459: Enhanced support to store runs on disk prior to uploading     them to OpenML.</li> <li>ADD #564: New helpers to access the structure of a flow (and find     its subflows).</li> <li>ADD #618: The software will from now on retry to connect to the     server if a connection failed. The number of retries can be     configured.</li> <li>FIX #538: Support loading clustering tasks.</li> <li>FIX #464: Fixes a bug related to listing functions (returns correct     listing size).</li> <li>FIX #580: Listing function now works properly when there are less     results than requested.</li> <li>FIX #571: Fixes an issue where tasks could not be downloaded in     parallel.</li> <li>FIX #536: Flows can now be printed when the flow name is None.</li> <li>FIX #504: Better support for hierarchical hyperparameters when     uploading scikit-learn\\'s grid and random search.</li> <li>FIX #569: Less strict checking of flow dependencies when loading     flows.</li> <li>FIX #431: Pickle of task splits are no longer cached.</li> <li>DOC #540: More examples for dataset uploading.</li> <li>DOC #554: Remove the doubled progress entry from the docs.</li> <li>MAINT #613: Utilize the latest updates in OpenML evaluation     listings.</li> <li>MAINT #482: Cleaner interface for handling search traces.</li> <li>MAINT #557: Continuous integration works for scikit-learn 0.18-0.20.</li> <li>MAINT #542: Continuous integration now runs python3.7 as well.</li> <li>MAINT #535: Continuous integration now enforces PEP8 compliance for     new code.</li> <li>MAINT #527: Replace deprecated nose by pytest.</li> <li>MAINT #510: Documentation is now built by travis-ci instead of     circle-ci.</li> <li>MAINT: Completely re-designed documentation built on sphinx gallery.</li> <li>MAINT #462: Appveyor CI support.</li> <li>MAINT #477: Improve error handling for issue     #479: the OpenML     connector fails earlier and with a better error message when failing     to create a flow from the OpenML description.</li> <li>MAINT #561: Improve documentation on running specific unit tests.</li> </ul>"},{"location":"python/progress/#04-07","title":"0.4.-0.7","text":"<p>There is no changelog for these versions.</p>"},{"location":"python/progress/#030","title":"0.3.0","text":"<ul> <li>Add this changelog</li> <li>2<sup>nd</sup> example notebook PyOpenML.ipynb</li> <li>Pagination support for list datasets and list tasks</li> </ul>"},{"location":"python/progress/#prior","title":"Prior","text":"<p>There is no changelog for prior versions.</p>"},{"location":"python/usage/","title":"User Guide","text":"<p>This document will guide you through the most important use cases, functions and classes in the OpenML Python API. Throughout this document, we will use pandas to format and filter tables.</p>"},{"location":"python/usage/#installation","title":"Installation","text":"<p>The OpenML Python package is a connector to OpenML. It allows you to use and share datasets and tasks, run machine learning algorithms on them and then share the results online.</p> <p>The \"intruduction tutorial and setup\" tutorial gives a short introduction on how to install and set up the OpenML Python connector, followed up by a simple example.</p>"},{"location":"python/usage/#configuration","title":"Configuration","text":"<p>The configuration file resides in a directory <code>.config/openml</code> in the home directory of the user and is called config (More specifically, it resides in the configuration directory specified by the XDGB Base Directory Specification). It consists of <code>key = value</code> pairs which are separated by newlines. The following keys are defined:</p> <ul> <li>apikey: required to access the server. The introduction tutorial describes how to obtain an API key.</li> <li>server: the server to connect to (default: <code>http://www.openml.org</code>).           For connection to the test server, set this to <code>test.openml.org</code>.</li> <li>cachedir: the root folder where the cache file directories should be created.     If not given, will default to <code>~/.openml/cache</code></li> <li>avoid_duplicate_runs: if set to <code>True</code> (default), when <code>run_flow_on_task</code> or similar methods             are called a lookup is performed to see if there already             exists such a run on the server. If so, download those             results instead.</li> <li> <p>retry_policy: Defines how to react when the server is unavailable or             experiencing high load. It determines both how often to             attempt to reconnect and how quickly to do so. Please don't             use <code>human</code> in an automated script that you run more than             one instance of, it might increase the time to complete your             jobs and that of others. One of:             -   human (default): For people running openml in interactive                 fashion. Try only a few times, but in quick succession.             -   robot: For people using openml in an automated fashion. Keep                 trying to reconnect for a longer time, quickly increasing                 the time between retries.</p> </li> <li> <p>connection_n_retries: number of times to retry a request if they fail.  Default depends on retry_policy (5 for <code>human</code>, 50 for <code>robot</code>)</p> </li> <li>verbosity: the level of output:<ul> <li>0: normal output</li> <li>1: info output</li> <li>2: debug output</li> </ul> </li> </ul> <p>This file is easily configurable by the <code>openml</code> command line interface. To see where the file is stored, and what its values are, use openml configure none. </p>"},{"location":"python/usage/#docker","title":"Docker","text":"<p>It is also possible to try out the latest development version of <code>openml-python</code> with docker:</p> <pre><code>docker run -it openml/openml-python\n</code></pre> <p>See the openml-python docker documentation for more information.</p>"},{"location":"python/usage/#key-concepts","title":"Key concepts","text":"<p>OpenML contains several key concepts which it needs to make machine learning research shareable. A machine learning experiment consists of one or several runs, which describe the performance of an algorithm (called a flow in OpenML), its hyperparameter settings (called a setup) on a task. A Task is the combination of a dataset, a split and an evaluation metric. In this user guide we will go through listing and exploring existing tasks to actually running machine learning algorithms on them. In a further user guide we will examine how to search through datasets in order to curate a list of tasks.</p> <p>A further explanation is given in the OpenML user guide.</p>"},{"location":"python/usage/#working-with-tasks","title":"Working with tasks","text":"<p>You can think of a task as an experimentation protocol, describing how to apply a machine learning model to a dataset in a way that is comparable with the results of others (more on how to do that further down). Tasks are containers, defining which dataset to use, what kind of task we\\'re solving (regression, classification, clustering, etc...) and which column to predict. Furthermore, it also describes how to split the dataset into a train and test set, whether to use several disjoint train and test splits (cross-validation) and whether this should be repeated several times. Also, the task defines a target metric for which a flow should be optimized.</p> <p>If you want to know more about tasks, try the \"Task tutorial\"</p>"},{"location":"python/usage/#running-machine-learning-algorithms-and-uploading-results","title":"Running machine learning algorithms and uploading results","text":"<p>In order to upload and share results of running a machine learning algorithm on a task, we need to create an openml.runs.OpenMLRun. A run object can be created by running a openml.flows.OpenMLFlow or a scikit-learn compatible model on a task. We will focus on the simpler example of running a scikit-learn model.</p> <p>Flows are descriptions of something runnable which does the machine learning. A flow contains all information to set up the necessary machine learning library and its dependencies as well as all possible parameters.</p> <p>A run is the outcome of running a flow on a task. It contains all parameter settings for the flow, a setup string (most likely a command line call) and all predictions of that run. When a run is uploaded to the server, the server automatically calculates several metrics which can be used to compare the performance of different flows to each other.</p> <p>So far, the OpenML Python connector works only with estimator objects following the scikit-learn estimator API. Those can be directly run on a task, and a flow will automatically be created or downloaded from the server if it already exists.</p> <p>See \"Simple Flows and Runs\" for a tutorial covers how to train different machine learning models, how to run machine learning models on OpenML data and how to share the results.</p>"},{"location":"python/usage/#datasets","title":"Datasets","text":"<p>OpenML provides a large collection of datasets and the benchmark OpenML100 which consists of a curated list of datasets.</p> <p>You can find the dataset that best fits your requirements by making use of the available metadata. The tutorial \"extended datasets\" which follows explains how to get a list of datasets, how to filter the list to find the dataset that suits your requirements and how to download a dataset.</p> <p>OpenML is about sharing machine learning results and the datasets they were obtained on. Learn how to share your datasets in the following tutorial \"Upload\" tutorial.</p>"},{"location":"python/usage/#extending-openml-python","title":"Extending OpenML-Python","text":"<p>OpenML-Python provides an extension interface to connect machine learning libraries directly to the API and ships a <code>scikit-learn</code> extension. Read more about them in the \"Extensions\" section.</p>"},{"location":"python/docker/readme/","title":"OpenML Python Container","text":"<p>This docker container has the latest version of openml-python downloaded and pre-installed. It can also be used by developers to run unit tests or build the docs in  a fresh and/or isolated unix environment.  This document contains information about:</p> <ol> <li>Usage: how to use the image and its main modes.</li> <li>Using local or remote code: useful when testing your own latest changes.</li> <li>Versions: identify which image to use.</li> <li>Development: information about the Docker image for developers.</li> </ol> <p>note: each docker image is shipped with a readme, which you can read with: <code>docker run --entrypoint=/bin/cat openml/openml-python:TAG readme.md</code></p>"},{"location":"python/docker/readme/#usage","title":"Usage","text":"<p>There are three main ways to use the image: running a pre-installed Python environment, running tests, and building documentation.</p>"},{"location":"python/docker/readme/#running-python-with-pre-installed-openml-python-default","title":"Running <code>Python</code> with pre-installed <code>OpenML-Python</code> (default):","text":"<p>To run <code>Python</code> with a pre-installed <code>OpenML-Python</code> environment run:</p> <pre><code>docker run -it openml/openml-python\n</code></pre> <p>this accepts the normal <code>Python</code> arguments, e.g.:</p> <pre><code>docker run openml/openml-python -c \"import openml; print(openml.__version__)\"\n</code></pre> <p>if you want to run a local script, it needs to be mounted first. Mount it into the <code>openml</code> folder:</p> <pre><code>docker run -v PATH/TO/FILE:/openml/MY_SCRIPT.py openml/openml-python MY_SCRIPT.py\n</code></pre>"},{"location":"python/docker/readme/#running-unit-tests","title":"Running unit tests","text":"<p>You can run the unit tests by passing <code>test</code> as the first argument. It also requires a local or remote repository to be specified, which is explained  [below]((#using-local-or-remote-code). For this example, we specify to test the <code>develop</code> branch:</p> <pre><code>docker run openml/openml-python test develop\n</code></pre>"},{"location":"python/docker/readme/#building-documentation","title":"Building documentation","text":"<p>You can build the documentation by passing <code>doc</code> as the first argument,  you should mount  an output directory in which the docs will be stored. You also need to provide a remote or local repository as explained in [the section below]((#using-local-or-remote-code). In this example, we build documentation for the <code>develop</code> branch. On Windows:</p> <pre><code>    docker run --mount type=bind,source=\"E:\\\\files/output\",destination=\"/output\" openml/openml-python doc develop\n</code></pre> <p>on Linux: <pre><code>    docker run --mount type=bind,source=\"./output\",destination=\"/output\" openml/openml-python doc develop\n</code></pre></p> <p>see [the section below]((#using-local-or-remote-code) for running against local changes or a remote branch.</p> <p>Note: you can forgo mounting an output directory to test if the docs build successfully, but the result will only be available within the docker container under <code>/openml/docs/build</code>.</p>"},{"location":"python/docker/readme/#using-local-or-remote-code","title":"Using local or remote code","text":"<p>You can build docs or run tests against your local repository or a Github repository. In the examples below, change the <code>source</code> to match the location of your local repository.</p>"},{"location":"python/docker/readme/#using-a-local-repository","title":"Using a local repository","text":"<p>To use a local directory, mount it in the <code>/code</code> directory,  on Windows:</p> <pre><code>    docker run --mount type=bind,source=\"E:\\\\repositories/openml-python\",destination=\"/code\" openml/openml-python test\n</code></pre> <p>on Linux: <pre><code>    docker run --mount type=bind,source=\"/Users/pietergijsbers/repositories/openml-python\",destination=\"/code\" openml/openml-python test\n</code></pre></p> <p>when building docs, you also need to mount an output directory as shown above, so add both:</p> <pre><code>docker run --mount type=bind,source=\"./output\",destination=\"/output\" --mount type=bind,source=\"/Users/pietergijsbers/repositories/openml-python\",destination=\"/code\" openml/openml-python doc\n</code></pre>"},{"location":"python/docker/readme/#using-a-github-repository","title":"Using a Github repository","text":"<p>Building from a remote repository requires you to specify a branch. The branch may be specified by name directly if it exists on the original repository (https://github.com/openml/openml-python/):</p> <pre><code>docker run --mount type=bind,source=PATH_TO_OUTPUT,destination=/output openml/openml-python [test,doc] BRANCH\n</code></pre> <p>Where <code>BRANCH</code> is the name of the branch for which to generate the documentation. It is also possible to build the documentation from the branch on a fork, in this case the <code>BRANCH</code> should be specified as <code>GITHUB_NAME#BRANCH</code> (e.g.  <code>PGijsbers#my_feature_branch</code>) and the name of the forked repository should be <code>openml-python</code>.</p>"},{"location":"python/docker/readme/#for-developers","title":"For developers","text":"<p>This section contains some notes about the structure of the image,  intended for those who want to work on it.</p>"},{"location":"python/docker/readme/#added-directories","title":"Added Directories","text":"<p>The <code>openml/openml-python</code> image is built on a vanilla <code>python:3</code> image. Additionally, it contains the following files are directories:</p> <ul> <li><code>/openml</code>: contains the openml-python repository in the state with which the image     was built by default. If working with a <code>BRANCH</code>, this repository will be set to     the <code>HEAD</code> of <code>BRANCH</code>.</li> <li><code>/openml/venv/</code>: contains the used virtual environment for <code>doc</code> and <code>test</code>. It has    <code>openml-python</code> dependencies pre-installed.  When invoked with <code>doc</code> or <code>test</code>, the     dependencies will be updated based on the <code>setup.py</code> of the <code>BRANCH</code> or mounted <code>/code</code>.</li> <li><code>/scripts/startup.sh</code>: the entrypoint of the image. Takes care of the automated features (e.g. <code>doc</code> and <code>test</code>).</li> </ul>"},{"location":"python/docker/readme/#building-the-image","title":"Building the image","text":"<p>To build the image yourself, execute <code>docker build -f Dockerfile .</code> from the <code>docker</code> directory of the <code>openml-python</code> repository. It will use the <code>startup.sh</code> as is, so any  local changes will be present in the image.</p>"},{"location":"python/docs/","title":"OpenML","text":"<p>Collaborative Machine Learning in Python</p> <p>Welcome to the documentation of the OpenML Python API, a connector to the collaborative machine learning platform OpenML.org. The OpenML Python package allows to use datasets and tasks from OpenML together with scikit-learn and share the results online.</p>"},{"location":"python/docs/#example","title":"Example","text":"<pre><code>import openml\nfrom sklearn import impute, tree, pipeline\n\n# Define a scikit-learn classifier or pipeline\nclf = pipeline.Pipeline(\n    steps=[\n        ('imputer', impute.SimpleImputer()),\n        ('estimator', tree.DecisionTreeClassifier())\n    ]\n)\n# Download the OpenML task for the pendigits dataset with 10-fold\n# cross-validation.\ntask = openml.tasks.get_task(32)\n# Run the scikit-learn model on the task.\nrun = openml.runs.run_model_on_task(clf, task)\n# Publish the experiment on OpenML (optional, requires an API key.\n# You can get your own API key by signing up to OpenML.org)\nrun.publish()\nprint(f'View the run online: {run.openml_url}')\n</code></pre> <p>Find more examples in the sidebar on the left.</p>"},{"location":"python/docs/#how-to-get-openml-for-python","title":"How to get OpenML for python","text":"<p>You can install the OpenML package via <code>pip</code> (we recommend using a virtual environment):</p> <pre><code>python -m pip install openml\n</code></pre> <p>For more advanced installation information, please see the \"Introduction\" example.</p>"},{"location":"python/docs/#further-information","title":"Further information","text":"<ul> <li>OpenML documentation</li> <li>OpenML client APIs</li> <li>OpenML developer guide</li> <li>Contact information</li> <li>Citation request</li> <li>OpenML blog</li> <li>OpenML twitter account</li> </ul>"},{"location":"python/docs/#contributing","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated. Please see the \"Contributing\" page for more information.</p>"},{"location":"python/docs/#citing-openml-python","title":"Citing OpenML-Python","text":"<p>If you use OpenML-Python in a scientific publication, we would appreciate a reference to our JMLR-MLOSS paper  \"OpenML-Python: an extensible Python API for OpenML\":</p> BibtexMLA <pre><code>@article{JMLR:v22:19-920,\n    author  = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas M\u00c3\u00bcller and Joaquin Vanschoren and Frank Hutter},\n    title   = {OpenML-Python: an extensible Python API for OpenML},\n    journal = {Journal of Machine Learning Research},\n    year    = {2021},\n    volume  = {22},\n    number  = {100},\n    pages   = {1--5},\n    url     = {http://jmlr.org/papers/v22/19-920.html}\n}\n</code></pre> <p>Feurer, Matthias, et al.  \"OpenML-Python: an extensible Python API for OpenML.\" Journal of Machine Learning Research 22.100 (2021):1\u22125.</p>"},{"location":"python/docs/contributing/","title":"Contributing","text":"<p>Contribution to the OpenML package is highly appreciated in all forms. In particular, a few ways to contribute to openml-python are:</p> <ul> <li>A direct contribution to the package, by means of improving the     code, documentation or examples. To get started, see this     file     with details on how to set up your environment to develop for     openml-python.</li> <li>A contribution to an openml-python extension. An extension package     allows OpenML to interface with a machine learning package (such     as scikit-learn or keras). These extensions are hosted in separate     repositories and may have their own guidelines. For more     information, see also extensions.</li> <li>Bug reports. If something doesn't work for you or is cumbersome,     please open a new issue to let us know about the problem. See     this     section.</li> <li>Cite OpenML if you use it in a     scientific publication.</li> <li>Visit one of our hackathons.</li> <li>Contribute to another OpenML project, such as the main OpenML     project.</li> </ul>"},{"location":"python/docs/extensions/","title":"Extensions","text":"<p>OpenML-Python provides an extension interface to connect other machine learning libraries than scikit-learn to OpenML. Please check the <code>api_extensions</code> and use the scikit-learn extension in <code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text role=\"class\"} as a starting point.</p>"},{"location":"python/docs/extensions/#list-of-extensions","title":"List of extensions","text":"<p>Here is a list of currently maintained OpenML extensions:</p> <ul> <li><code>openml.extensions.sklearn.SklearnExtension</code>{.interpreted-text     role=\"class\"}</li> <li>openml-keras</li> <li>openml-pytorch</li> <li>openml-tensorflow (for tensorflow     2+)</li> </ul>"},{"location":"python/docs/extensions/#connecting-new-machine-learning-libraries","title":"Connecting new machine learning libraries","text":""},{"location":"python/docs/extensions/#content-of-the-library","title":"Content of the Library","text":"<p>To leverage support from the community and to tap in the potential of OpenML, interfacing with popular machine learning libraries is essential. The OpenML-Python package is capable of downloading meta-data and results (data, flows, runs), regardless of the library that was used to upload it. However, in order to simplify the process of uploading flows and runs from a specific library, an additional interface can be built. The OpenML-Python team does not have the capacity to develop and maintain such interfaces on its own. For this reason, we have built an extension interface to allows others to contribute back. Building a suitable extension for therefore requires an understanding of the current OpenML-Python support.</p> <p>The <code>sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py</code>{.interpreted-text role=\"ref\"} tutorial shows how scikit-learn currently works with OpenML-Python as an extension. The sklearn extension packaged with the openml-python repository can be used as a template/benchmark to build the new extension.</p>"},{"location":"python/docs/extensions/#api","title":"API","text":"<ul> <li>The extension scripts must import the [openml]{.title-ref} package     and be able to interface with any function from the OpenML-Python     <code>api</code>.</li> <li>The extension has to be defined as a Python class and must inherit     from <code>openml.extensions.Extension</code>.</li> <li>This class needs to have all the functions from [class     Extension]{.title-ref} overloaded as required.</li> <li>The redefined functions should have adequate and appropriate     docstrings. The [Sklearn Extension API     :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref}     is a good example to follow.</li> </ul>"},{"location":"python/docs/extensions/#interfacing-with-openml-python","title":"Interfacing with OpenML-Python","text":"<p>Once the new extension class has been defined, the openml-python module to <code>openml.extensions.register_extension</code> must be called to allow OpenML-Python to interface the new extension.</p> <p>The following methods should get implemented. Although the documentation in the [Extension]{.title-ref} interface should always be leading, here we list some additional information and best practices. The [Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is a good example to follow. Note that most methods are relatively simple and can be implemented in several lines of code.</p> <ul> <li>General setup (required)<ul> <li><code>can_handle_flow</code>: Takes as     argument an OpenML flow, and checks whether this can be handled     by the current extension. The OpenML database consists of many     flows, from various workbenches (e.g., scikit-learn, Weka, mlr).     This method is called before a model is being deserialized.     Typically, the flow-dependency field is used to check whether     the specific library is present, and no unknown libraries are     present there.</li> <li><code>can_handle_model</code>: Similar as     <code>can_handle_flow</code>, except that in     this case a Python object is given. As such, in many cases, this     method can be implemented by checking whether this adheres to a     certain base class.</li> </ul> </li> <li>Serialization and De-serialization (required)<ul> <li><code>flow_to_model</code>: deserializes the     OpenML Flow into a model (if the library can indeed handle the     flow). This method has an important interplay with     <code>model_to_flow</code>. Running these     two methods in succession should result in exactly the same     model (or flow). This property can be used for unit testing     (e.g., build a model with hyperparameters, make predictions on a     task, serialize it to a flow, deserialize it back, make it     predict on the same task, and check whether the predictions are     exactly the same.) The example in the scikit-learn interface     might seem daunting, but note that here some complicated design     choices were made, that allow for all sorts of interesting     research questions. It is probably good practice to start easy.</li> <li><code>model_to_flow</code>: The inverse of     <code>flow_to_model</code>. Serializes a     model into an OpenML Flow. The flow should preserve the class,     the library version, and the tunable hyperparameters.</li> <li><code>get_version_information</code>: Return     a tuple with the version information of the important libraries.</li> <li><code>create_setup_string</code>: No longer     used, and will be deprecated soon.</li> </ul> </li> <li>Performing runs (required)<ul> <li><code>is_estimator</code>: Gets as input a     class, and checks whether it has the status of estimator in the     library (typically, whether it has a train method and a predict     method).</li> <li><code>seed_model</code>: Sets a random seed     to the model.</li> <li><code>_run_model_on_fold</code>: One of the     main requirements for a library to generate run objects for the     OpenML server. Obtains a train split (with labels) and a test     split (without labels) and the goal is to train a model on the     train split and return the predictions on the test split. On top     of the actual predictions, also the class probabilities should     be determined. For classifiers that do not return class     probabilities, this can just be the hot-encoded predicted label.     The predictions will be evaluated on the OpenML server. Also,     additional information can be returned, for example,     user-defined measures (such as runtime information, as this can     not be inferred on the server). Additionally, information about     a hyperparameter optimization trace can be provided.</li> <li><code>obtain_parameter_values</code>:     Obtains the hyperparameters of a given model and the current     values. Please note that in the case of a hyperparameter     optimization procedure (e.g., random search), you only should     return the hyperparameters of this procedure (e.g., the     hyperparameter grid, budget, etc) and that the chosen model will     be inferred from the optimization trace.</li> <li><code>check_if_model_fitted</code>: Check     whether the train method of the model has been called (and as     such, whether the predict method can be used).</li> </ul> </li> <li>Hyperparameter optimization (optional)<ul> <li><code>instantiate_model_from_hpo_class</code>{.interpreted-text     role=\"meth\"}: If a given run has recorded the hyperparameter     optimization trace, then this method can be used to     reinstantiate the model with hyperparameters of a given     hyperparameter optimization iteration. Has some similarities     with <code>flow_to_model</code> (as this     method also sets the hyperparameters of a model). Note that     although this method is required, it is not necessary to     implement any logic if hyperparameter optimization is not     implemented. Simply raise a [NotImplementedError]{.title-ref}     then.</li> </ul> </li> </ul>"},{"location":"python/docs/extensions/#hosting-the-library","title":"Hosting the library","text":"<p>Each extension created should be a stand-alone repository, compatible with the OpenML-Python repository. The extension repository should work off-the-shelf with OpenML-Python installed.</p> <p>Create a public Github repo with the following directory structure:</p> <pre><code>| [repo name]\n|    |-- [extension name]\n|    |    |-- __init__.py\n|    |    |-- extension.py\n|    |    |-- config.py (optionally)\n</code></pre>"},{"location":"python/docs/extensions/#recommended","title":"Recommended","text":"<ul> <li>Test cases to keep the extension up to date with the     [openml-python]{.title-ref} upstream changes.</li> <li>Documentation of the extension API, especially if any new     functionality added to OpenML-Python\\'s extension design.</li> <li>Examples to show how the new extension interfaces and works with     OpenML-Python.</li> <li>Create a PR to add the new extension to the OpenML-Python API     documentation.</li> </ul> <p>Happy contributing!</p>"},{"location":"python/docs/progress/","title":"Changelog","text":""},{"location":"python/docs/progress/#next","title":"next","text":"<ul> <li>MAINT #1340: Add Numpy 2.0 support. Update tests to work with     scikit-learn \\&lt;= 1.5.</li> <li>ADD #1342: Add HTTP header to requests to indicate they are from     openml-python.</li> </ul>"},{"location":"python/docs/progress/#0142","title":"0.14.2","text":"<ul> <li>MAINT #1280: Use the server-provided <code>parquet_url</code> instead of     <code>minio_url</code> to determine the location of the parquet file.</li> <li>ADD #716: add documentation for remaining attributes of classes     and functions.</li> <li>ADD #1261: more annotations for type hints.</li> <li>MAINT #1294: update tests to new tag specification.</li> <li>FIX #1314: Update fetching a bucket from MinIO.</li> <li>FIX #1315: Make class label retrieval more lenient.</li> <li>ADD #1316: add feature descriptions ontologies support.</li> <li>MAINT #1310/#1307: switch to ruff and resolve all mypy errors.</li> </ul>"},{"location":"python/docs/progress/#0141","title":"0.14.1","text":"<ul> <li>FIX: Fallback on downloading ARFF when failing to download parquet     from MinIO due to a ServerError.</li> </ul>"},{"location":"python/docs/progress/#0140","title":"0.14.0","text":"<p>IMPORTANT: This release paves the way towards a breaking update of OpenML-Python. From version 0.15, functions that had the option to return a pandas DataFrame will return a pandas DataFrame by default. This version (0.14) emits a warning if you still use the old access functionality. More concretely:</p> <ul> <li>In 0.15 we will drop the ability to return dictionaries in listing     calls and only provide pandas DataFrames. To disable warnings in     0.14 you have to request a pandas DataFrame (using     <code>output_format=\"dataframe\"</code>).</li> <li>In 0.15 we will drop the ability to return datasets as numpy arrays     and only provide pandas DataFrames. To disable warnings in 0.14 you     have to request a pandas DataFrame (using     <code>dataset_format=\"dataframe\"</code>).</li> </ul> <p>Furthermore, from version 0.15, OpenML-Python will no longer download datasets and dataset metadata by default. This version (0.14) emits a warning if you don\\'t explicitly specifiy the desired behavior.</p> <p>Please see the pull requests #1258 and #1260 for further information.</p> <ul> <li>ADD #1081: New flag that allows disabling downloading dataset     features.</li> <li>ADD #1132: New flag that forces a redownload of cached data.</li> <li>FIX #1244: Fixes a rare bug where task listing could fail when the     server returned invalid data.</li> <li>DOC #1229: Fixes a comment string for the main example.</li> <li>DOC #1241: Fixes a comment in an example.</li> <li>MAINT #1124: Improve naming of helper functions that govern the     cache directories.</li> <li>MAINT #1223, #1250: Update tools used in pre-commit to the latest     versions (<code>black==23.30</code>, <code>mypy==1.3.0</code>, <code>flake8==6.0.0</code>).</li> <li>MAINT #1253: Update the citation request to the JMLR paper.</li> <li>MAINT #1246: Add a warning that warns the user that checking for     duplicate runs on the server cannot be done without an API key.</li> </ul>"},{"location":"python/docs/progress/#0131","title":"0.13.1","text":"<ul> <li>ADD #1081 #1132: Add additional options for (not) downloading     datasets <code>openml.datasets.get_dataset</code> and cache management.</li> <li>ADD #1028: Add functions to delete runs, flows, datasets, and tasks     (e.g., <code>openml.datasets.delete_dataset</code>).</li> <li>ADD #1144: Add locally computed results to the <code>OpenMLRun</code> object\\'s     representation if the run was created locally and not downloaded     from the server.</li> <li>ADD #1180: Improve the error message when the checksum of a     downloaded dataset does not match the checksum provided by the API.</li> <li>ADD #1201: Make <code>OpenMLTraceIteration</code> a dataclass.</li> <li>DOC #1069: Add argument documentation for the <code>OpenMLRun</code> class.</li> <li>DOC #1241 #1229 #1231: Minor documentation fixes and resolve     documentation examples not working.</li> <li>FIX #1197 #559 #1131: Fix the order of ground truth and predictions     in the <code>OpenMLRun</code> object and in <code>format_prediction</code>.</li> <li>FIX #1198: Support numpy 1.24 and higher.</li> <li>FIX #1216: Allow unknown task types on the server. This is only     relevant when new task types are added to the test server.</li> <li>FIX #1223: Fix mypy errors for implicit optional typing.</li> <li>MAINT #1155: Add dependabot github action to automatically update     other github actions.</li> <li>MAINT #1199: Obtain pre-commit\\'s flake8 from github.com instead of     gitlab.com.</li> <li>MAINT #1215: Support latest numpy version.</li> <li>MAINT #1218: Test Python3.6 on Ubuntu 20.04 instead of the latest     Ubuntu (which is 22.04).</li> <li>MAINT #1221 #1212 #1206 #1211: Update github actions to the latest     versions.</li> </ul>"},{"location":"python/docs/progress/#0130","title":"0.13.0","text":"<ul> <li>FIX #1030: <code>pre-commit</code> hooks now no longer should issue a     warning.</li> <li>FIX #1058, #1100: Avoid <code>NoneType</code> error when printing task     without <code>class_labels</code> attribute.</li> <li>FIX #1110: Make arguments to <code>create_study</code> and <code>create_suite</code>     that are defined as optional by the OpenML XSD actually optional.</li> <li>FIX #1147: <code>openml.flow.flow_exists</code> no longer requires an API     key.</li> <li>FIX #1184: Automatically resolve proxies when downloading from     minio. Turn this off by setting environment variable     <code>no_proxy=\"*\"</code>.</li> <li>MAINT #1088: Do CI for Windows on Github Actions instead of     Appveyor.</li> <li>MAINT #1104: Fix outdated docstring for <code>list_task</code>.</li> <li>MAINT #1146: Update the pre-commit dependencies.</li> <li>ADD #1103: Add a <code>predictions</code> property to OpenMLRun for easy     accessibility of prediction data.</li> <li>ADD #1188: EXPERIMENTAL. Allow downloading all files from a minio     bucket with <code>download_all_files=True</code> for <code>get_dataset</code>.</li> </ul>"},{"location":"python/docs/progress/#0122","title":"0.12.2","text":"<ul> <li>ADD #1065: Add a <code>retry_policy</code> configuration option that determines     the frequency and number of times to attempt to retry server     requests.</li> <li>ADD #1075: A docker image is now automatically built on a push to     develop. It can be used to build docs or run tests in an isolated     environment.</li> <li>ADD: You can now avoid downloading \\'qualities\\' meta-data when     downloading a task with the <code>download_qualities</code> parameter of     <code>openml.tasks.get_task[s]</code> functions.</li> <li>DOC: Fixes a few broken links in the documentation.</li> <li>DOC #1061: Improve examples to always show a warning when they     switch to the test server.</li> <li>DOC #1067: Improve documentation on the scikit-learn extension     interface.</li> <li>DOC #1068: Create dedicated extensions page.</li> <li>FIX #1075: Correctly convert [y]{.title-ref} to a pandas series when     downloading sparse data.</li> <li>MAINT: Rename [master]{.title-ref} brach to [ main]{.title-ref}     branch.</li> <li>MAINT/DOC: Automatically check for broken external links when     building the documentation.</li> <li>MAINT/DOC: Fail documentation building on warnings. This will make     the documentation building fail if a reference cannot be found (i.e.     an internal link is broken).</li> </ul>"},{"location":"python/docs/progress/#0121","title":"0.12.1","text":"<ul> <li>ADD #895/#1038: Measure runtimes of scikit-learn runs also for     models which are parallelized via the joblib.</li> <li>DOC #1050: Refer to the webpage instead of the XML file in the main     example.</li> <li>DOC #1051: Document existing extensions to OpenML-Python besides the     shipped scikit-learn extension.</li> <li>FIX #1035: Render class attributes and methods again.</li> <li>ADD #1049: Add a command line tool for configuration openml-python.</li> <li>FIX #1042: Fixes a rare concurrency issue with OpenML-Python and     joblib which caused the joblib worker pool to fail.</li> <li>FIX #1053: Fixes a bug which could prevent importing the package in     a docker container.</li> </ul>"},{"location":"python/docs/progress/#0120","title":"0.12.0","text":"<ul> <li>ADD #964: Validate <code>ignore_attribute</code>, <code>default_target_attribute</code>,     <code>row_id_attribute</code> are set to attributes that exist on the dataset     when calling <code>create_dataset</code>.</li> <li>ADD #979: Dataset features and qualities are now also cached in     pickle format.</li> <li>ADD #982: Add helper functions for column transformers.</li> <li>ADD #989: <code>run_model_on_task</code> will now warn the user the the model     passed has already been fitted.</li> <li>ADD #1009 : Give possibility to not download the dataset qualities.     The cached version is used even so download attribute is false.</li> <li>ADD #1016: Add scikit-learn 0.24 support.</li> <li>ADD #1020: Add option to parallelize evaluation of tasks with     joblib.</li> <li>ADD #1022: Allow minimum version of dependencies to be listed for a     flow, use more accurate minimum versions for scikit-learn     dependencies.</li> <li>ADD #1023: Add admin-only calls for adding topics to datasets.</li> <li>ADD #1029: Add support for fetching dataset from a minio server in     parquet format.</li> <li>ADD #1031: Generally improve runtime measurements, add them for some     previously unsupported flows (e.g. BaseSearchCV derived flows).</li> <li>DOC #973 : Change the task used in the welcome page example so it no     longer fails using numerical dataset.</li> <li>MAINT #671: Improved the performance of <code>check_datasets_active</code> by     only querying the given list of datasets in contrast to querying all     datasets. Modified the corresponding unit test.</li> <li>MAINT #891: Changed the way that numerical features are stored.     Numerical features that range from 0 to 255 are now stored as uint8,     which reduces the storage space required as well as storing and     loading times.</li> <li>MAINT #975, #988: Add CI through Github Actions.</li> <li>MAINT #977: Allow <code>short</code> and <code>long</code> scenarios for unit tests.     Reduce the workload for some unit tests.</li> <li>MAINT #985, #1000: Improve unit test stability and output     readability, and adds load balancing.</li> <li>MAINT #1018: Refactor data loading and storage. Data is now     compressed on the first call to [get_data]{.title-ref}.</li> <li>MAINT #1024: Remove flaky decorator for study unit test.</li> <li>FIX #883 #884 #906 #972: Various improvements to the caching system.</li> <li>FIX #980: Speed up <code>check_datasets_active</code>.</li> <li>FIX #984: Add a retry mechanism when the server encounters a     database issue.</li> <li>FIX #1004: Fixed an issue that prevented installation on some     systems (e.g. Ubuntu).</li> <li>FIX #1013: Fixes a bug where <code>OpenMLRun.setup_string</code> was not     uploaded to the server, prepares for <code>run_details</code> being sent from     the server.</li> <li>FIX #1021: Fixes an issue that could occur when running unit tests     and openml-python was not in PATH.</li> <li>FIX #1037: Fixes a bug where a dataset could not be loaded if a     categorical value had listed nan-like as a possible category.</li> </ul>"},{"location":"python/docs/progress/#0110","title":"0.11.0","text":"<ul> <li>ADD #753: Allows uploading custom flows to OpenML via OpenML-Python.</li> <li>ADD #777: Allows running a flow on pandas dataframes (in addition to     numpy arrays).</li> <li>ADD #888: Allow passing a [task_id]{.title-ref} to     [run_model_on_task]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #929: Add <code>edit_dataset</code> and <code>fork_dataset</code> to allow editing and     forking of uploaded datasets.</li> <li>ADD #866, #943: Add support for scikit-learn\\'s     [passthrough]{.title-ref} and [drop]{.title-ref} when uploading     flows to OpenML.</li> <li>ADD #879: Add support for scikit-learn\\'s MLP hyperparameter     [layer_sizes]{.title-ref}.</li> <li>ADD #894: Support caching of datasets using feather format as an     option.</li> <li>ADD #945: PEP 561 compliance for distributing Type information.</li> <li>DOC #660: Remove nonexistent argument from docstring.</li> <li>DOC #901: The API reference now documents the config file and its     options.</li> <li>DOC #912: API reference now shows [create_task]{.title-ref}.</li> <li>DOC #954: Remove TODO text from documentation.</li> <li>DOC #960: document how to upload multiple ignore attributes.</li> <li>FIX #873: Fixes an issue which resulted in incorrect URLs when     printing OpenML objects after switching the server.</li> <li>FIX #885: Logger no longer registered by default. Added utility     functions to easily register logging to console and file.</li> <li>FIX #890: Correct the scaling of data in the SVM example.</li> <li>MAINT #371: <code>list_evaluations</code> default <code>size</code> changed from <code>None</code> to     <code>10_000</code>.</li> <li>MAINT #767: Source distribution installation is now unit-tested.</li> <li>MAINT #781: Add pre-commit and automated code formatting with black.</li> <li>MAINT #804: Rename arguments of list_evaluations to indicate they     expect lists of ids.</li> <li>MAINT #836: OpenML supports only pandas version 1.0.0 or above.</li> <li>MAINT #865: OpenML no longer bundles test files in the source     distribution.</li> <li>MAINT #881: Improve the error message for too-long URIs.</li> <li>MAINT #897: Dropping support for Python 3.5.</li> <li>MAINT #916: Adding support for Python 3.8.</li> <li>MAINT #920: Improve error messages for dataset upload.</li> <li>MAINT #921: Improve hangling of the OpenML server URL in the config     file.</li> <li>MAINT #925: Improve error handling and error message when loading     datasets.</li> <li>MAINT #928: Restructures the contributing documentation.</li> <li>MAINT #936: Adding support for scikit-learn 0.23.X.</li> <li>MAINT #945: Make OpenML-Python PEP562 compliant.</li> <li>MAINT #951: Converts TaskType class to a TaskType enum.</li> </ul>"},{"location":"python/docs/progress/#0102","title":"0.10.2","text":"<ul> <li>ADD #857: Adds task type ID to list_runs</li> <li>DOC #862: Added license BSD 3-Clause to each of the source files.</li> </ul>"},{"location":"python/docs/progress/#0101","title":"0.10.1","text":"<ul> <li>ADD #175: Automatically adds the docstring of scikit-learn objects     to flow and its parameters.</li> <li>ADD #737: New evaluation listing call that includes the     hyperparameter settings.</li> <li>ADD #744: It is now possible to only issue a warning and not raise     an exception if the package versions for a flow are not met when     deserializing it.</li> <li>ADD #783: The URL to download the predictions for a run is now     stored in the run object.</li> <li>ADD #790: Adds the uploader name and id as new filtering options for     <code>list_evaluations</code>.</li> <li>ADD #792: New convenience function <code>openml.flow.get_flow_id</code>.</li> <li>ADD #861: Debug-level log information now being written to a file in     the cache directory (at most 2 MB).</li> <li>DOC #778: Introduces instructions on how to publish an extension to     support other libraries than scikit-learn.</li> <li>DOC #785: The examples section is completely restructured into     simple simple examples, advanced examples and examples showcasing     the use of OpenML-Python to reproduce papers which were done with     OpenML-Python.</li> <li>DOC #788: New example on manually iterating through the split of a     task.</li> <li>DOC #789: Improve the usage of dataframes in the examples.</li> <li>DOC #791: New example for the paper Efficient and Robust Automated     Machine Learning by Feurer et al. (2015).</li> <li>DOC #803: New example for the paper Don't Rule Out Simple Models     Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear     Classifiers in OpenML by Benjamin Strang et al. (2018).</li> <li>DOC #808: New example demonstrating basic use cases of a dataset.</li> <li>DOC #810: New example demonstrating the use of benchmarking studies     and suites.</li> <li>DOC #832: New example for the paper Scalable Hyperparameter     Transfer Learning by Valerio Perrone et al. (2019)</li> <li>DOC #834: New example showing how to plot the loss surface for a     support vector machine.</li> <li>FIX #305: Do not require the external version in the flow XML when     loading an object.</li> <li>FIX #734: Better handling of \\\"old\\\" flows.</li> <li>FIX #736: Attach a StreamHandler to the openml logger instead of the     root logger.</li> <li>FIX #758: Fixes an error which made the client API crash when     loading a sparse data with categorical variables.</li> <li>FIX #779: Do not fail on corrupt pickle</li> <li>FIX #782: Assign the study id to the correct class attribute.</li> <li>FIX #819: Automatically convert column names to type string when     uploading a dataset.</li> <li>FIX #820: Make <code>__repr__</code> work for datasets which do not have an id.</li> <li>MAINT #796: Rename an argument to make the function     <code>list_evaluations</code> more consistent.</li> <li>MAINT #811: Print the full error message given by the server.</li> <li>MAINT #828: Create base class for OpenML entity classes.</li> <li>MAINT #829: Reduce the number of data conversion warnings.</li> <li>MAINT #831: Warn if there\\'s an empty flow description when     publishing a flow.</li> <li>MAINT #837: Also print the flow XML if a flow fails to validate.</li> <li>FIX #838: Fix list_evaluations_setups to work when evaluations are     not a 100 multiple.</li> <li>FIX #847: Fixes an issue where the client API would crash when     trying to download a dataset when there are no qualities available     on the server.</li> <li>MAINT #849: Move logic of most different <code>publish</code> functions into     the base class.</li> <li>MAINt #850: Remove outdated test code.</li> </ul>"},{"location":"python/docs/progress/#0100","title":"0.10.0","text":"<ul> <li>ADD #737: Add list_evaluations_setups to return hyperparameters     along with list of evaluations.</li> <li>FIX #261: Test server is cleared of all files uploaded during unit     testing.</li> <li>FIX #447: All files created by unit tests no longer persist in     local.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>FIX #447: All files created by unit tests are deleted after the     completion of all unit tests.</li> <li>FIX #589: Fixing a bug that did not successfully upload the columns     to ignore when creating and publishing a dataset.</li> <li>FIX #608: Fixing dataset_id referenced before assignment error in     get_run function.</li> <li>DOC #639: More descriptive documention for function to convert array     format.</li> <li>DOC #719: Add documentation on uploading tasks.</li> <li>ADD #687: Adds a function to retrieve the list of evaluation     measures available.</li> <li>ADD #695: A function to retrieve all the data quality measures     available.</li> <li>ADD #412: Add a function to trim flow names for scikit-learn flows.</li> <li>ADD #715: [list_evaluations]{.title-ref} now has an option to sort     evaluations by score (value).</li> <li>ADD #722: Automatic reinstantiation of flow in     [run_model_on_task]{.title-ref}. Clearer errors if that\\'s not     possible.</li> <li>ADD #412: The scikit-learn extension populates the short name field     for flows.</li> <li>MAINT #726: Update examples to remove deprecation warnings from     scikit-learn</li> <li>MAINT #752: Update OpenML-Python to be compatible with sklearn 0.21</li> <li>ADD #790: Add user ID and name to list_evaluations</li> </ul>"},{"location":"python/docs/progress/#090","title":"0.9.0","text":"<ul> <li>ADD #560: OpenML-Python can now handle regression tasks as well.</li> <li>ADD #620, #628, #632, #649, #682: Full support for studies and     distinguishes suites from studies.</li> <li>ADD #607: Tasks can now be created and uploaded.</li> <li>ADD #647, #673: Introduced the extension interface. This provides an     easy way to create a hook for machine learning packages to perform     e.g. automated runs.</li> <li>ADD #548, #646, #676: Support for Pandas DataFrame and     SparseDataFrame</li> <li>ADD #662: Results of listing functions can now be returned as     pandas.DataFrame.</li> <li>ADD #59: Datasets can now also be retrieved by name.</li> <li>ADD #672: Add timing measurements for runs, when possible.</li> <li>ADD #661: Upload time and error messages now displayed with     [list_runs]{.title-ref}.</li> <li>ADD #644: Datasets can now be downloaded \\'lazily\\', retrieving only     metadata at first, and the full dataset only when necessary.</li> <li>ADD #659: Lazy loading of task splits.</li> <li>ADD #516: [run_flow_on_task]{.title-ref} flow uploading is now     optional.</li> <li>ADD #680: Adds     [openml.config.start_using_configuration_for_example]{.title-ref}     (and resp. stop) to easily connect to the test server.</li> <li>ADD #75, #653: Adds a pretty print for objects of the top-level     classes.</li> <li>FIX #642: [check_datasets_active]{.title-ref} now correctly also     returns active status of deactivated datasets.</li> <li>FIX #304, #636: Allow serialization of numpy datatypes and list of     lists of more types (e.g. bools, ints) for flows.</li> <li>FIX #651: Fixed a bug that would prevent openml-python from finding     the user\\'s config file.</li> <li>FIX #693: OpenML-Python uses liac-arff instead of scipy.io for     loading task splits now.</li> <li>DOC #678: Better color scheme for code examples in documentation.</li> <li>DOC #681: Small improvements and removing list of missing functions.</li> <li>DOC #684: Add notice to examples that connect to the test server.</li> <li>DOC #688: Add new example on retrieving evaluations.</li> <li>DOC #691: Update contributing guidelines to use Github draft feature     instead of tags in title.</li> <li>DOC #692: All functions are documented now.</li> <li>MAINT #184: Dropping Python2 support.</li> <li>MAINT #596: Fewer dependencies for regular pip install.</li> <li>MAINT #652: Numpy and Scipy are no longer required before     installation.</li> <li>MAINT #655: Lazy loading is now preferred in unit tests.</li> <li>MAINT #667: Different tag functions now share code.</li> <li>MAINT #666: More descriptive error message for     [TypeError]{.title-ref} in [list_runs]{.title-ref}.</li> <li>MAINT #668: Fix some type hints.</li> <li>MAINT #677: [dataset.get_data]{.title-ref} now has consistent     behavior in its return type.</li> <li>MAINT #686: Adds ignore directives for several [mypy]{.title-ref}     folders.</li> <li>MAINT #629, #630: Code now adheres to single PEP8 standard.</li> </ul>"},{"location":"python/docs/progress/#080","title":"0.8.0","text":"<ul> <li>ADD #440: Improved dataset upload.</li> <li>ADD #545, #583: Allow uploading a dataset from a pandas DataFrame.</li> <li>ADD #528: New functions to update the status of a dataset.</li> <li>ADD #523: Support for scikit-learn 0.20\\'s new ColumnTransformer.</li> <li>ADD #459: Enhanced support to store runs on disk prior to uploading     them to OpenML.</li> <li>ADD #564: New helpers to access the structure of a flow (and find     its subflows).</li> <li>ADD #618: The software will from now on retry to connect to the     server if a connection failed. The number of retries can be     configured.</li> <li>FIX #538: Support loading clustering tasks.</li> <li>FIX #464: Fixes a bug related to listing functions (returns correct     listing size).</li> <li>FIX #580: Listing function now works properly when there are less     results than requested.</li> <li>FIX #571: Fixes an issue where tasks could not be downloaded in     parallel.</li> <li>FIX #536: Flows can now be printed when the flow name is None.</li> <li>FIX #504: Better support for hierarchical hyperparameters when     uploading scikit-learn\\'s grid and random search.</li> <li>FIX #569: Less strict checking of flow dependencies when loading     flows.</li> <li>FIX #431: Pickle of task splits are no longer cached.</li> <li>DOC #540: More examples for dataset uploading.</li> <li>DOC #554: Remove the doubled progress entry from the docs.</li> <li>MAINT #613: Utilize the latest updates in OpenML evaluation     listings.</li> <li>MAINT #482: Cleaner interface for handling search traces.</li> <li>MAINT #557: Continuous integration works for scikit-learn 0.18-0.20.</li> <li>MAINT #542: Continuous integration now runs python3.7 as well.</li> <li>MAINT #535: Continuous integration now enforces PEP8 compliance for     new code.</li> <li>MAINT #527: Replace deprecated nose by pytest.</li> <li>MAINT #510: Documentation is now built by travis-ci instead of     circle-ci.</li> <li>MAINT: Completely re-designed documentation built on sphinx gallery.</li> <li>MAINT #462: Appveyor CI support.</li> <li>MAINT #477: Improve error handling for issue     #479: the OpenML     connector fails earlier and with a better error message when failing     to create a flow from the OpenML description.</li> <li>MAINT #561: Improve documentation on running specific unit tests.</li> </ul>"},{"location":"python/docs/progress/#04-07","title":"0.4.-0.7","text":"<p>There is no changelog for these versions.</p>"},{"location":"python/docs/progress/#030","title":"0.3.0","text":"<ul> <li>Add this changelog</li> <li>2<sup>nd</sup> example notebook PyOpenML.ipynb</li> <li>Pagination support for list datasets and list tasks</li> </ul>"},{"location":"python/docs/progress/#prior","title":"Prior","text":"<p>There is no changelog for prior versions.</p>"},{"location":"python/docs/usage/","title":"User Guide","text":"<p>This document will guide you through the most important use cases, functions and classes in the OpenML Python API. Throughout this document, we will use pandas to format and filter tables.</p>"},{"location":"python/docs/usage/#installation","title":"Installation","text":"<p>The OpenML Python package is a connector to OpenML. It allows you to use and share datasets and tasks, run machine learning algorithms on them and then share the results online.</p> <p>The \"intruduction tutorial and setup\" tutorial gives a short introduction on how to install and set up the OpenML Python connector, followed up by a simple example.</p>"},{"location":"python/docs/usage/#configuration","title":"Configuration","text":"<p>The configuration file resides in a directory <code>.config/openml</code> in the home directory of the user and is called config (More specifically, it resides in the configuration directory specified by the XDGB Base Directory Specification). It consists of <code>key = value</code> pairs which are separated by newlines. The following keys are defined:</p> <ul> <li>apikey: required to access the server. The introduction tutorial describes how to obtain an API key.</li> <li>server: the server to connect to (default: <code>http://www.openml.org</code>).           For connection to the test server, set this to <code>test.openml.org</code>.</li> <li>cachedir: the root folder where the cache file directories should be created.     If not given, will default to <code>~/.openml/cache</code></li> <li>avoid_duplicate_runs: if set to <code>True</code> (default), when <code>run_flow_on_task</code> or similar methods             are called a lookup is performed to see if there already             exists such a run on the server. If so, download those             results instead.</li> <li> <p>retry_policy: Defines how to react when the server is unavailable or             experiencing high load. It determines both how often to             attempt to reconnect and how quickly to do so. Please don't             use <code>human</code> in an automated script that you run more than             one instance of, it might increase the time to complete your             jobs and that of others. One of:             -   human (default): For people running openml in interactive                 fashion. Try only a few times, but in quick succession.             -   robot: For people using openml in an automated fashion. Keep                 trying to reconnect for a longer time, quickly increasing                 the time between retries.</p> </li> <li> <p>connection_n_retries: number of times to retry a request if they fail.  Default depends on retry_policy (5 for <code>human</code>, 50 for <code>robot</code>)</p> </li> <li>verbosity: the level of output:<ul> <li>0: normal output</li> <li>1: info output</li> <li>2: debug output</li> </ul> </li> </ul> <p>This file is easily configurable by the <code>openml</code> command line interface. To see where the file is stored, and what its values are, use openml configure none. </p>"},{"location":"python/docs/usage/#docker","title":"Docker","text":"<p>It is also possible to try out the latest development version of <code>openml-python</code> with docker:</p> <pre><code>docker run -it openml/openml-python\n</code></pre> <p>See the openml-python docker documentation for more information.</p>"},{"location":"python/docs/usage/#key-concepts","title":"Key concepts","text":"<p>OpenML contains several key concepts which it needs to make machine learning research shareable. A machine learning experiment consists of one or several runs, which describe the performance of an algorithm (called a flow in OpenML), its hyperparameter settings (called a setup) on a task. A Task is the combination of a dataset, a split and an evaluation metric. In this user guide we will go through listing and exploring existing tasks to actually running machine learning algorithms on them. In a further user guide we will examine how to search through datasets in order to curate a list of tasks.</p> <p>A further explanation is given in the OpenML user guide.</p>"},{"location":"python/docs/usage/#working-with-tasks","title":"Working with tasks","text":"<p>You can think of a task as an experimentation protocol, describing how to apply a machine learning model to a dataset in a way that is comparable with the results of others (more on how to do that further down). Tasks are containers, defining which dataset to use, what kind of task we\\'re solving (regression, classification, clustering, etc...) and which column to predict. Furthermore, it also describes how to split the dataset into a train and test set, whether to use several disjoint train and test splits (cross-validation) and whether this should be repeated several times. Also, the task defines a target metric for which a flow should be optimized.</p> <p>If you want to know more about tasks, try the \"Task tutorial\"</p>"},{"location":"python/docs/usage/#running-machine-learning-algorithms-and-uploading-results","title":"Running machine learning algorithms and uploading results","text":"<p>In order to upload and share results of running a machine learning algorithm on a task, we need to create an openml.runs.OpenMLRun. A run object can be created by running a openml.flows.OpenMLFlow or a scikit-learn compatible model on a task. We will focus on the simpler example of running a scikit-learn model.</p> <p>Flows are descriptions of something runnable which does the machine learning. A flow contains all information to set up the necessary machine learning library and its dependencies as well as all possible parameters.</p> <p>A run is the outcome of running a flow on a task. It contains all parameter settings for the flow, a setup string (most likely a command line call) and all predictions of that run. When a run is uploaded to the server, the server automatically calculates several metrics which can be used to compare the performance of different flows to each other.</p> <p>So far, the OpenML Python connector works only with estimator objects following the scikit-learn estimator API. Those can be directly run on a task, and a flow will automatically be created or downloaded from the server if it already exists.</p> <p>See \"Simple Flows and Runs\" for a tutorial covers how to train different machine learning models, how to run machine learning models on OpenML data and how to share the results.</p>"},{"location":"python/docs/usage/#datasets","title":"Datasets","text":"<p>OpenML provides a large collection of datasets and the benchmark OpenML100 which consists of a curated list of datasets.</p> <p>You can find the dataset that best fits your requirements by making use of the available metadata. The tutorial \"extended datasets\" which follows explains how to get a list of datasets, how to filter the list to find the dataset that suits your requirements and how to download a dataset.</p> <p>OpenML is about sharing machine learning results and the datasets they were obtained on. Learn how to share your datasets in the following tutorial \"Upload\" tutorial.</p>"},{"location":"python/docs/usage/#extending-openml-python","title":"Extending OpenML-Python","text":"<p>OpenML-Python provides an extension interface to connect machine learning libraries directly to the API and ships a <code>scikit-learn</code> extension. Read more about them in the \"Extensions\" section.</p>"},{"location":"pytorch/","title":"Pytorch extension for OpenML python","text":"<p>Pytorch extension for openml-python API.</p>"},{"location":"pytorch/#installation-instructions","title":"Installation Instructions:","text":"<p><code>pip install openml-pytorch</code></p> <p>PyPi link https://pypi.org/project/openml-pytorch/</p>"},{"location":"pytorch/#usage","title":"Usage","text":"<p>To use this extension, you need to have a task from OpenML. You can either browse the OpenML website to find a task (and get it's ID), or follow the example to create a task from a custom dataset.</p> <p>Then, follow one of the examples in the Examples folder to see how to use this extension for your type of data.</p> <p>Import openML libraries <pre><code>import torch.nn\nimport torch.optim\n\nimport openml_pytorch.config\nimport openml\nimport logging\n\nfrom openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda\nimport torchvision\nfrom openml_pytorch.trainer import convert_to_rgb\n</code></pre> Create a pytorch model and get a task from openML <pre><code>model = torchvision.models.efficientnet_b0(num_classes=200)\n# Download the OpenML task for tiniest imagenet\ntask = openml.tasks.get_task(362128)\n</code></pre> Download the task from openML and define Data and Trainer configuration <pre><code>transform = Compose(\n    [\n        ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n        Lambda(\n            convert_to_rgb\n        ),  # Convert PIL Image to RGB if it's not already.\n        Resize(\n            (64, 64)\n        ),  # Resize the image.\n        ToTensor(),  # Convert the PIL Image back to a tensor.\n    ]\n)\ndata_module = OpenMLDataModule(\n    type_of_data=\"image\",\n    file_dir=\"datasets\",\n    filename_col=\"image_path\",\n    target_mode=\"categorical\",\n    target_column=\"label\",\n    batch_size = 64,\n    transform=transform\n)\ntrainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n)\nopenml_pytorch.config.trainer = trainer\n</code></pre> Run the model on the task <pre><code>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\nrun.publish()\nprint('URL for run: %s/run/%d' % (openml.config.server, run.run_id))\n</code></pre> Note: The input layer of the network should be compatible with OpenML data output shape. Please check examples for more information.</p> <p>Additionally, if you want to publish the run with onnx file, then you must call <code>openml_pytorch.add_onnx_to_run()</code> immediately before <code>run.publish()</code>. </p> <pre><code>run = openml_pytorch.add_onnx_to_run(run)\n</code></pre>"},{"location":"pytorch/Limitations%20of%20the%20API/","title":"Limitations","text":"<ul> <li>Image datasets are supported as a workaround by using a CSV file with image paths. This is not ideal and might eventually be replaced by something else. At the moment, the focus is on tabular data.</li> <li>Many features (like custom metrics, models etc) are still dependant on the OpenML Python API, which is in the middle of a major rewrite. Until that is complete, this package will not be able to provide all the features it aims to.</li> </ul>"},{"location":"pytorch/Philosophy%20behind%20the%20API%20Design/","title":"Philosophy behind the API design","text":"<p>This API is designed to make it easier to use PyTorch with OpenML and has been heavily inspired by the current state of the art Deep Learning frameworks like FastAI and PyTorch Lightning. </p> <p>To make the library as modular as possible, callbacks are used throughout the training loop. This allows for easy customization of the training loop without having to modify the core code.</p>"},{"location":"pytorch/Philosophy%20behind%20the%20API%20Design/#separation-of-concerns","title":"Separation of Concerns","text":"<p>Here, we focus on the data, model and training as separate blocks that can be strung together in a pipeline. This makes it easier to experiment with different models, data and training strategies.</p> <p>That being the case, the OpenMLDataModule and OpenMLTrainerModule are designed to handle the data and training respectively. This might seem a bit verbose at first, but it makes it easier to understand what is happening at each step of the process and allows for easier customization.</p>"},{"location":"pytorch/API%20reference/Callbacks/","title":"Callbacks","text":"<p>Callbacks module contains classes and functions for handling callback functions during an event-driven process. This makes it easier to customize the behavior of the training loop and add additional functionality to the training process without modifying the core code.</p> <p>To use a callback, create a class that inherits from the Callback class and implement the necessary methods. Callbacks can be used to perform actions at different stages of the training process, such as at the beginning or end of an epoch, batch, or fitting process. Then pass the callback object to the Trainer.</p>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.AvgStats","title":"<code>AvgStats</code>","text":"<p>AvgStats class is used to track and accumulate average statistics (like loss and other metrics) during training and validation phases.</p> <p>Attributes:     metrics (list): A list of metric functions to be tracked.     in_train (bool): A flag to indicate if the statistics are for the training phase.</p> <p>Methods:     init(metrics, in_train):         Initializes the AvgStats with metrics and in_train flag.</p> <pre><code>reset():\n    Resets the accumulated statistics.\n\nall_stats:\n    Property that returns all accumulated statistics including loss and metrics.\n\navg_stats:\n    Property that returns the average of the accumulated statistics.\n\naccumulate(run):\n    Accumulates the statistics using the data from the given run.\n\n__repr__():\n    Returns a string representation of the average statistics.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class AvgStats:\n    \"\"\"\n    AvgStats class is used to track and accumulate average statistics (like loss and other metrics) during training and validation phases.\n\n    Attributes:\n        metrics (list): A list of metric functions to be tracked.\n        in_train (bool): A flag to indicate if the statistics are for the training phase.\n\n    Methods:\n        __init__(metrics, in_train):\n            Initializes the AvgStats with metrics and in_train flag.\n\n        reset():\n            Resets the accumulated statistics.\n\n        all_stats:\n            Property that returns all accumulated statistics including loss and metrics.\n\n        avg_stats:\n            Property that returns the average of the accumulated statistics.\n\n        accumulate(run):\n            Accumulates the statistics using the data from the given run.\n\n        __repr__():\n            Returns a string representation of the average statistics.\n    \"\"\"\n    def __init__(self, metrics, in_train):\n        self.metrics, self.in_train = listify(metrics), in_train\n\n    def reset(self):\n        self.tot_loss, self.count = 0.0, 0\n        self.tot_mets = [0.0] * len(self.metrics)\n\n    @property\n    def all_stats(self):\n        return [self.tot_loss.item()] + self.tot_mets\n\n    @property\n    def avg_stats(self):\n        return [o / self.count for o in self.all_stats]\n\n    def accumulate(self, run):\n        bn = run.xb.shape[0]\n        self.tot_loss += run.loss * bn\n        self.count += bn\n        for i, m in enumerate(self.metrics):\n            self.tot_mets[i] += m(run.pred, run.yb) * bn\n\n    def __repr__(self):\n        if not self.count:\n            return \"\"\n        return f\"{'train' if self.in_train else 'valid'}: {self.avg_stats}\"\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.AvgStatsCallBack","title":"<code>AvgStatsCallBack</code>","text":"<p>               Bases: <code>Callback</code></p> <p>AvgStatsCallBack class is a custom callback used to track and print average statistics for training and validation phases during the training loop.</p> <p>Arguments:     metrics: A list of metric functions to evaluate during training and validation.</p> <p>Methods:     init: Initializes the callback with given metrics and sets up AvgStats objects for both training and validation phases.     begin_epoch: Resets the statistics at the beginning of each epoch.     after_loss: Accumulates the metrics after computing the loss, differentiating between training and validation phases.     after_epoch: Prints the accumulated statistics for both training and validation phases after each epoch.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class AvgStatsCallBack(Callback):\n    \"\"\"\n    AvgStatsCallBack class is a custom callback used to track and print average statistics for training and validation phases during the training loop.\n\n    Arguments:\n        metrics: A list of metric functions to evaluate during training and validation.\n\n    Methods:\n        __init__: Initializes the callback with given metrics and sets up AvgStats objects for both training and validation phases.\n        begin_epoch: Resets the statistics at the beginning of each epoch.\n        after_loss: Accumulates the metrics after computing the loss, differentiating between training and validation phases.\n        after_epoch: Prints the accumulated statistics for both training and validation phases after each epoch.\n    \"\"\"\n    def __init__(self, metrics):\n        self.train_stats, self.valid_stats = AvgStats(metrics, True), AvgStats(\n            metrics, False\n        )\n\n    def begin_epoch(self):\n        self.train_stats.reset()\n        self.valid_stats.reset()\n\n    def after_loss(self):\n        stats = self.train_stats if self.in_train else self.valid_stats\n        with torch.no_grad():\n            stats.accumulate(self.run)\n\n    def after_epoch(self):\n        print(self.train_stats)\n        print(self.valid_stats)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Callback","title":"<code>Callback</code>","text":"<p>Callback class is a base class designed for handling different callback functions during an event-driven process. It provides functionality to set a runner, retrieve the class name in snake_case format, directly call callback methods, and delegate attribute access to the runner if the attribute does not exist in the Callback class.</p> <p>The _order is used to decide the order of Callbacks.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class Callback:\n    \"\"\"\n\n        Callback class is a base class designed for handling different callback functions during\n        an event-driven process. It provides functionality to set a runner, retrieve the class\n        name in snake_case format, directly call callback methods, and delegate attribute access\n        to the runner if the attribute does not exist in the Callback class.\n\n        The _order is used to decide the order of Callbacks.\n\n    \"\"\"\n    _order = 0\n\n    def set_runner(self, run) -&gt; None:\n        self.run = run\n\n    @property\n    def name(self):\n        name = re.sub(r\"Callback$\", \"\", self.__class__.__name__)\n        return camel2snake(name or \"callback\")\n\n    def __call__(self, cb_name):\n        f = getattr(self, cb_name, None)\n        if f and f():\n            return True\n        return False\n\n    def __getattr__(self, k):\n        return getattr(self.run, k)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler","title":"<code>ParamScheduler</code>","text":"<p>               Bases: <code>Callback</code></p> <p>Manages scheduling of parameter adjustments over the course of training.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class ParamScheduler(Callback):\n    \"\"\"\n    Manages scheduling of parameter adjustments over the course of training.\n    \"\"\"\n    _order = 1\n\n    def __init__(self, pname, sched_funcs):\n        self.pname, self.sched_funcs = pname, sched_funcs\n\n    def begin_fit(self):\n        \"\"\"\n        Prepare the scheduler at the start of the fitting process.\n        This method ensures that sched_funcs is a list with one function per parameter group.\n        \"\"\"\n        if not isinstance(self.sched_funcs, (list, tuple)):\n            self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)\n\n    def set_param(self):\n        \"\"\"\n        Adjust the parameter value for each parameter group based on the scheduling function.\n        Ensures the number of scheduling functions matches the number of parameter groups.\n        \"\"\"\n        assert len(self.opt.param_groups) == len(self.sched_funcs)\n        for pg, f in zip(self.opt.param_groups, self.sched_funcs):\n            pg[self.pname] = f(self.n_epochs / self.epochs)\n\n    def begin_batch(self):\n        \"\"\"\n        Apply parameter adjustments at the beginning of each batch if in training mode.\n        \"\"\"\n        if self.in_train:\n            self.set_param()\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler.begin_batch","title":"<code>begin_batch()</code>","text":"<p>Apply parameter adjustments at the beginning of each batch if in training mode.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def begin_batch(self):\n    \"\"\"\n    Apply parameter adjustments at the beginning of each batch if in training mode.\n    \"\"\"\n    if self.in_train:\n        self.set_param()\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler.begin_fit","title":"<code>begin_fit()</code>","text":"<p>Prepare the scheduler at the start of the fitting process. This method ensures that sched_funcs is a list with one function per parameter group.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def begin_fit(self):\n    \"\"\"\n    Prepare the scheduler at the start of the fitting process.\n    This method ensures that sched_funcs is a list with one function per parameter group.\n    \"\"\"\n    if not isinstance(self.sched_funcs, (list, tuple)):\n        self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.ParamScheduler.set_param","title":"<code>set_param()</code>","text":"<p>Adjust the parameter value for each parameter group based on the scheduling function. Ensures the number of scheduling functions matches the number of parameter groups.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def set_param(self):\n    \"\"\"\n    Adjust the parameter value for each parameter group based on the scheduling function.\n    Ensures the number of scheduling functions matches the number of parameter groups.\n    \"\"\"\n    assert len(self.opt.param_groups) == len(self.sched_funcs)\n    for pg, f in zip(self.opt.param_groups, self.sched_funcs):\n        pg[self.pname] = f(self.n_epochs / self.epochs)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder","title":"<code>Recorder</code>","text":"<p>               Bases: <code>Callback</code></p> <p>Recorder is a callback class used to record learning rates and losses during the training process.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class Recorder(Callback):\n    \"\"\"\n        Recorder is a callback class used to record learning rates and losses during the training process.\n    \"\"\"\n    def begin_fit(self):\n        \"\"\"\n        Initializes attributes necessary for the fitting process.\n\n        Sets up learning rates and losses storage.\n\n        Attributes:\n            self.lrs (list): A list of lists, where each inner list will hold learning rates for a parameter group.\n            self.losses (list): An empty list to store loss values during the fitting process.\n        \"\"\"\n        self.lrs = [[] for _ in self.opt.param_groups]\n        self.losses = []\n\n    def after_batch(self):\n        \"\"\"\n        Handles operations to execute after each training batch.\n\n        Modifies the learning rate for each parameter group in the optimizer \n        and appends the current learning rate and loss to the corresponding lists.\n\n        \"\"\"\n        if not self.in_train:\n            return\n        for pg, lr in zip(self.opt.param_groups, self.lrs):\n            lr.append(pg[\"lr\"])\n        self.losses.append(self.loss.detach().cpu())\n\n    def plot_lr(self, pgid=-1):\n        \"\"\"\n        Plots the learning rate for a given parameter group.\n        \"\"\"\n        plt.plot(self.lrs[pgid])\n\n    def plot_loss(self, skip_last=0):\n        \"\"\"\n        Plots the loss for a given parameter group.\n        \"\"\"\n        plt.plot(self.losses[: len(self.losses) - skip_last])\n\n    def plot(self, skip_last=0, pgid=-1):\n        \"\"\"\n        Generates a plot of the loss values against the learning rates.\n        \"\"\"\n        losses = [o.item() for o in self.losses]\n        lrs = self.lrs[pgid]\n        n = len(losses) - skip_last\n        plt.xscale(\"log\")\n        plt.plot(lrs[:n], losses[:n])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.after_batch","title":"<code>after_batch()</code>","text":"<p>Handles operations to execute after each training batch.</p> <p>Modifies the learning rate for each parameter group in the optimizer  and appends the current learning rate and loss to the corresponding lists.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def after_batch(self):\n    \"\"\"\n    Handles operations to execute after each training batch.\n\n    Modifies the learning rate for each parameter group in the optimizer \n    and appends the current learning rate and loss to the corresponding lists.\n\n    \"\"\"\n    if not self.in_train:\n        return\n    for pg, lr in zip(self.opt.param_groups, self.lrs):\n        lr.append(pg[\"lr\"])\n    self.losses.append(self.loss.detach().cpu())\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.begin_fit","title":"<code>begin_fit()</code>","text":"<p>Initializes attributes necessary for the fitting process.</p> <p>Sets up learning rates and losses storage.</p> <p>Attributes:     self.lrs (list): A list of lists, where each inner list will hold learning rates for a parameter group.     self.losses (list): An empty list to store loss values during the fitting process.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def begin_fit(self):\n    \"\"\"\n    Initializes attributes necessary for the fitting process.\n\n    Sets up learning rates and losses storage.\n\n    Attributes:\n        self.lrs (list): A list of lists, where each inner list will hold learning rates for a parameter group.\n        self.losses (list): An empty list to store loss values during the fitting process.\n    \"\"\"\n    self.lrs = [[] for _ in self.opt.param_groups]\n    self.losses = []\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.plot","title":"<code>plot(skip_last=0, pgid=-1)</code>","text":"<p>Generates a plot of the loss values against the learning rates.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def plot(self, skip_last=0, pgid=-1):\n    \"\"\"\n    Generates a plot of the loss values against the learning rates.\n    \"\"\"\n    losses = [o.item() for o in self.losses]\n    lrs = self.lrs[pgid]\n    n = len(losses) - skip_last\n    plt.xscale(\"log\")\n    plt.plot(lrs[:n], losses[:n])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.plot_loss","title":"<code>plot_loss(skip_last=0)</code>","text":"<p>Plots the loss for a given parameter group.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def plot_loss(self, skip_last=0):\n    \"\"\"\n    Plots the loss for a given parameter group.\n    \"\"\"\n    plt.plot(self.losses[: len(self.losses) - skip_last])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.Recorder.plot_lr","title":"<code>plot_lr(pgid=-1)</code>","text":"<p>Plots the learning rate for a given parameter group.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def plot_lr(self, pgid=-1):\n    \"\"\"\n    Plots the learning rate for a given parameter group.\n    \"\"\"\n    plt.plot(self.lrs[pgid])\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.TrainEvalCallback","title":"<code>TrainEvalCallback</code>","text":"<p>               Bases: <code>Callback</code></p> <p>TrainEvalCallback class is a custom callback used during the training and validation phases of a machine learning model to perform specific actions at the beginning and after certain events.</p> <p>Methods:</p> <p>begin_fit():     Initialize the number of epochs and iteration counts at the start     of the fitting process.</p> <p>after_batch():     Update the epoch and iteration counts after each batch during     training.</p> <p>begin_epoch():     Set the current epoch, switch the model to training mode, and     indicate that the model is in training.</p> <p>begin_validate():     Switch the model to evaluation mode and indicate that the model     is in validation.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>class TrainEvalCallback(Callback):\n    \"\"\"\n        TrainEvalCallback class is a custom callback used during the training\n        and validation phases of a machine learning model to perform specific\n        actions at the beginning and after certain events.\n\n        Methods:\n\n        begin_fit():\n            Initialize the number of epochs and iteration counts at the start\n            of the fitting process.\n\n        after_batch():\n            Update the epoch and iteration counts after each batch during\n            training.\n\n        begin_epoch():\n            Set the current epoch, switch the model to training mode, and\n            indicate that the model is in training.\n\n        begin_validate():\n            Switch the model to evaluation mode and indicate that the model\n            is in validation.\n    \"\"\"\n    def begin_fit(self):\n        self.run.n_epochs = 0\n        self.run.n_iter = 0\n\n    def after_batch(self):\n        if not self.in_train:\n            return\n        self.run.n_epochs += 1.0 / self.iters\n        self.run.n_iter += 1\n\n    def begin_epoch(self):\n        self.run.n_epochs = self.epoch\n        self.model.train()\n        self.run.in_train = True\n\n    def begin_validate(self):\n        self.model.eval()\n        self.run.in_train = False\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.annealer","title":"<code>annealer(f)</code>","text":"<p>A decorator function for creating a partially applied function with predefined start and end arguments. The inner function <code>_inner</code> captures the <code>start</code> and <code>end</code> parameters and returns a <code>partial</code> object that fixes these parameters for the decorated function <code>f</code>.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def annealer(f) -&gt; callable:\n    \"\"\"\n    A decorator function for creating a partially applied function with predefined start and end arguments.\n    The inner function `_inner` captures the `start` and `end` parameters and returns a `partial` object that fixes these parameters for the decorated function `f`.\n    \"\"\"\n    def _inner(start, end):\n        return partial(f, start, end)\n\n    return _inner\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.camel2snake","title":"<code>camel2snake(name)</code>","text":"<p>Convert <code>name</code> from camel case to snake case.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def camel2snake(name : str) -&gt; str:\n    \"\"\"\n    Convert `name` from camel case to snake case.\n    \"\"\"\n    s1 = re.sub(_camel_re1, r\"\\1_\\2\", name)\n    return re.sub(_camel_re2, r\"\\1_\\2\", s1).lower()\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.combine_scheds","title":"<code>combine_scheds(pcts, scheds)</code>","text":"<p>Combine multiple scheduling functions.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def combine_scheds(pcts: Iterable[float], scheds: Iterable[callable]) -&gt; callable:\n    \"\"\"\n    Combine multiple scheduling functions.\n    \"\"\"\n    assert sum(pcts) == 1.0\n    pcts = torch.tensor([0] + listify(pcts))\n    assert torch.all(pcts &gt;= 0)\n    pcts = torch.cumsum(pcts, 0)\n\n    def _inner(pos):\n        idx = (pos &gt;= pcts).nonzero().max()\n        actual_pos = (pos - pcts[idx]) / (pcts[idx + 1] - pcts[idx])\n        return scheds[idx](actual_pos)\n\n    return _inner\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.listify","title":"<code>listify(o=None)</code>","text":"<p>Convert <code>o</code> to list. If <code>o</code> is None, return empty list.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>def listify(o = None) -&gt; list:\n    \"\"\"\n    Convert `o` to list. If `o` is None, return empty list.\n    \"\"\"\n    if o is None:\n        return []\n    if isinstance(o, list):\n        return o\n    if isinstance(o, str):\n        return [o]\n    if isinstance(o, Iterable):\n        return list(o)\n    return [o]\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_cos","title":"<code>sched_cos(start, end, pos)</code>","text":"<p>A cosine schedule function.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_cos(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    A cosine schedule function.\n    \"\"\"\n    return start + (1 + math.cos(math.pi * (1 - pos))) * (end - start) / 2\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_exp","title":"<code>sched_exp(start, end, pos)</code>","text":"<p>Exponential schedule function.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_exp(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    Exponential schedule function.\n    \"\"\"\n    return start * (end / start) ** pos\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_lin","title":"<code>sched_lin(start, end, pos)</code>","text":"<p>A linear schedule function.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_lin(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    A linear schedule function.\n    \"\"\"\n    return start + pos * (end - start)\n</code></pre>"},{"location":"pytorch/API%20reference/Callbacks/#callbacks.sched_no","title":"<code>sched_no(start, end, pos)</code>","text":"<p>Disabled scheduling.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/callbacks.py</code> <pre><code>@annealer\ndef sched_no(start: float, end: float, pos: float) -&gt; float:\n    \"\"\"\n    Disabled scheduling.\n    \"\"\"\n    return start\n</code></pre>"},{"location":"pytorch/API%20reference/Custom%20Datasets/","title":"Custom Datasets","text":"<p>This module contains custom dataset classes for handling image and tabular data from OpenML in PyTorch. To add support for new data types, new classes can be added to this module.</p>"},{"location":"pytorch/API%20reference/Custom%20Datasets/#custom_datasets.OpenMLImageDataset","title":"<code>OpenMLImageDataset</code>","text":"<p>               Bases: <code>Dataset</code></p> <p>Class representing an image dataset from OpenML for use in PyTorch.</p> <p>Methods:</p> <pre><code>__init__(self, X, y, image_size, image_dir, transform_x=None, transform_y=None)\n    Initializes the dataset with given data, image size, directory, and optional transformations.\n\n__getitem__(self, idx)\n    Retrieves an image and its corresponding label (if available) from the dataset at the specified index. Applies transformations if provided.\n\n__len__(self)\n    Returns the total number of images in the dataset.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/custom_datasets.py</code> <pre><code>class OpenMLImageDataset(Dataset):\n    \"\"\"\n        Class representing an image dataset from OpenML for use in PyTorch.\n\n        Methods:\n\n            __init__(self, X, y, image_size, image_dir, transform_x=None, transform_y=None)\n                Initializes the dataset with given data, image size, directory, and optional transformations.\n\n            __getitem__(self, idx)\n                Retrieves an image and its corresponding label (if available) from the dataset at the specified index. Applies transformations if provided.\n\n            __len__(self)\n                Returns the total number of images in the dataset.\n    \"\"\"\n    def __init__(self, X, y, image_size, image_dir, transform_x = None, transform_y = None):\n        self.X = X\n        self.y = y\n        self.image_size = image_size\n        self.image_dir = image_dir\n        self.transform_x = transform_x\n        self.transform_y = transform_y\n\n    def __getitem__(self, idx):\n        img_name = str(os.path.join(self.image_dir, self.X.iloc[idx, 0]))\n        image = read_image(img_name)\n        image = image.float()\n        image = T.Resize((self.image_size, self.image_size))(image)\n        if self.transform_x is not None:\n            image = self.transform_x(image)\n        if self.y is not None:\n            label = self.y.iloc[idx]\n            if label is not None:\n                if self.transform_y is not None:\n                    label = self.transform_y(label)\n                return image, label\n        else:\n            return image\n\n    def __len__(self):\n        return len(self.X)\n</code></pre>"},{"location":"pytorch/API%20reference/Custom%20Datasets/#custom_datasets.OpenMLTabularDataset","title":"<code>OpenMLTabularDataset</code>","text":"<p>               Bases: <code>Dataset</code></p> <p>OpenMLTabularDataset</p> <p>A custom dataset class to handle tabular data from OpenML (or any similar tabular dataset). It encodes categorical features and the target column using LabelEncoder from sklearn.</p> <p>Methods:     init(X, y) : Initializes the dataset with the data and the target column.                      Encodes the categorical features and target if provided.</p> <pre><code>__getitem__(idx): Retrieves the input data and target value at the specified index.\n                  Converts the data to tensors and returns them.\n\n__len__(): Returns the length of the dataset.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/custom_datasets.py</code> <pre><code>class OpenMLTabularDataset(Dataset):\n    \"\"\"\n    OpenMLTabularDataset\n\n    A custom dataset class to handle tabular data from OpenML (or any similar tabular dataset).\n    It encodes categorical features and the target column using LabelEncoder from sklearn.\n\n    Methods:\n        __init__(X, y) : Initializes the dataset with the data and the target column.\n                         Encodes the categorical features and target if provided.\n\n        __getitem__(idx): Retrieves the input data and target value at the specified index.\n                          Converts the data to tensors and returns them.\n\n        __len__(): Returns the length of the dataset.\n    \"\"\"\n    def __init__(self, X, y):\n        self.data = X\n        # self.target_col_name = target_col\n        for col in self.data.select_dtypes(include=['object', 'category']):\n            # convert to float\n            self.data[col] = self.data[col].astype('category').cat.codes\n        self.label_mapping = None\n\n        # self.label_mapping = preprocessing.LabelEncoder()\n        # try:\n        #     self.data = self.data.apply(self.label_mapping.fit_transform)\n        # except ValueError:\n        #     pass\n\n        # try:\n        #     self.y = self.label_mapping.fit_transform(y)\n        # except ValueError:\n        #     self.y = None\n        self.y = y\n\n    def __getitem__(self, idx):\n        # x is the input data, y is the target value from the target column\n        x = self.data.iloc[idx, :]\n        x = torch.tensor(x.values.astype('float32'))\n        if self.y is not None:\n            y = self.y[idx]\n            y = torch.tensor(y)\n            return x, y\n        else:\n            return x\n\n\n    def __len__(self):\n        return len(self.data)\n</code></pre>"},{"location":"pytorch/API%20reference/Metrics/","title":"Metrics","text":"<p>This module provides utility functions for evaluating model performance and activation functions. It includes functions to compute the accuracy, top-k accuracy of model predictions, and the sigmoid function.</p>"},{"location":"pytorch/API%20reference/Metrics/#metrics.accuracy","title":"<code>accuracy(out, yb)</code>","text":"<p>Computes the accuracy of model predictions.</p> <p>Parameters: out (Tensor): The output tensor from the model, containing predicted class scores. yb (Tensor): The ground truth labels tensor.</p> <p>Returns: Tensor: The mean accuracy of the predictions, computed as a float tensor.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/metrics.py</code> <pre><code>def accuracy(out, yb):\n    \"\"\"\n\n    Computes the accuracy of model predictions.\n\n    Parameters:\n    out (Tensor): The output tensor from the model, containing predicted class scores.\n    yb (Tensor): The ground truth labels tensor.\n\n    Returns:\n    Tensor: The mean accuracy of the predictions, computed as a float tensor.\n    \"\"\"\n    return (torch.argmax(out, dim=1) == yb.long()).float().mean()\n</code></pre>"},{"location":"pytorch/API%20reference/Metrics/#metrics.accuracy_topk","title":"<code>accuracy_topk(out, yb, k=5)</code>","text":"<p>Computes the top-k accuracy of the given model outputs.</p> <p>Args:     out (torch.Tensor): The output predictions of the model, of shape (batch_size, num_classes).     yb (torch.Tensor): The ground truth labels, of shape (batch_size,).     k (int, optional): The number of top predictions to consider. Default is 5.</p> <p>Returns:     float: The top-k accuracy as a float value.</p> <p>The function calculates how often the true label is among the top-k predicted labels.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/metrics.py</code> <pre><code>def accuracy_topk(out, yb, k=5):\n    \"\"\"\n\n    Computes the top-k accuracy of the given model outputs.\n\n    Args:\n        out (torch.Tensor): The output predictions of the model, of shape (batch_size, num_classes).\n        yb (torch.Tensor): The ground truth labels, of shape (batch_size,).\n        k (int, optional): The number of top predictions to consider. Default is 5.\n\n    Returns:\n        float: The top-k accuracy as a float value.\n\n    The function calculates how often the true label is among the top-k predicted labels.\n    \"\"\"\n    return (torch.topk(out, k, dim=1)[1] == yb.long().unsqueeze(1)).float().mean()\n</code></pre>"},{"location":"pytorch/API%20reference/Metrics/#metrics.sigmoid","title":"<code>sigmoid(x)</code>","text":"<p>Computes the sigmoid function</p> <p>The sigmoid function is defined as 1 / (1 + exp(-x)). This function is used to map any real-valued number into the range (0, 1). It is widely used in machine learning, especially in logistic regression and neural networks.</p> <p>Args:     x (numpy.ndarray or float): The input value or array over which the     sigmoid function should be applied.</p> <p>Returns:     numpy.ndarray or float: The sigmoid of the input value or array.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/metrics.py</code> <pre><code>def sigmoid(x):\n    \"\"\"\n    Computes the sigmoid function\n\n    The sigmoid function is defined as 1 / (1 + exp(-x)). This function is used\n    to map any real-valued number into the range (0, 1). It is widely used in\n    machine learning, especially in logistic regression and neural networks.\n\n    Args:\n        x (numpy.ndarray or float): The input value or array over which the\n        sigmoid function should be applied.\n\n    Returns:\n        numpy.ndarray or float: The sigmoid of the input value or array.\n    \"\"\"\n    return 1 / (1 + np.exp(-x))\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/","title":"OpenML Connection","text":"<p>This module defines the Pytorch extension for OpenML-python.</p>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension","title":"<code>PytorchExtension</code>","text":"<p>               Bases: <code>Extension</code></p> <p>Connect Pytorch to OpenML-Python.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>class PytorchExtension(Extension):\n    \"\"\"Connect Pytorch to OpenML-Python.\"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    def can_handle_flow(cls, flow: 'OpenMLFlow') -&gt; bool:\n        \"\"\"Check whether a given describes a Pytorch estimator.\n\n        This is done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return cls._is_pytorch_flow(flow)\n\n    @classmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model is an instance of ``torch.nn.Module``.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        from torch.nn import Module\n        return isinstance(model, Module)\n\n    ################################################################################################\n    # Method for dataloader \n\n\n\n    ################################################################################################\n    # Methods for flow serialization and de-serialization\n\n    def flow_to_model(self, flow: 'OpenMLFlow', initialize_with_defaults: bool = False) -&gt; Any:\n        \"\"\"Initializes a Pytorch model based on a flow.\n\n        Parameters\n        ----------\n        flow : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        return self._deserialize_pytorch(flow, initialize_with_defaults=initialize_with_defaults)\n\n    def _deserialize_pytorch(\n        self,\n        o: Any,\n        components: Optional[Dict] = None,\n        initialize_with_defaults: bool = False,\n        recursion_depth: int = 0,\n    ) -&gt; Any:\n        \"\"\"Recursive function to deserialize a Pytorch flow.\n\n        This function delegates all work to the respective functions to deserialize special data\n        structures etc.\n\n        Parameters\n        ----------\n        o : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        components : dict\n\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        recursion_depth : int\n            The depth at which this flow is called, mostly for debugging\n            purposes\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n\n        logging.info('-%s flow_to_pytorch START o=%s, components=%s, '\n                     'init_defaults=%s' % ('-' * recursion_depth, o, components,\n                                           initialize_with_defaults))\n        depth_pp = recursion_depth + 1  # shortcut var, depth plus plus\n\n        # First, we need to check whether the presented object is a json string.\n        # JSON strings are used to encoder parameter values. By passing around\n        # json strings for parameters, we make sure that we can flow_to_pytorch\n        # the parameter values to the correct type.\n\n        if isinstance(o, str):\n            try:\n                o = json.loads(o)\n            except JSONDecodeError:\n                pass\n\n        if isinstance(o, dict):\n            # Check if the dict encodes a 'special' object, which could not\n            # easily converted into a string, but rather the information to\n            # re-create the object were stored in a dictionary.\n            if 'oml-python:serialized_object' in o:\n                serialized_type = o['oml-python:serialized_object']\n                value = o['value']\n                if serialized_type == 'type':\n                    rval = self._deserialize_type(value)\n                elif serialized_type == 'function':\n                    rval = self._deserialize_function(value)\n                elif serialized_type == 'methoddescriptor':\n                    rval = self._deserialize_methoddescriptor(value)\n                elif serialized_type == 'component_reference':\n                    assert components is not None  # Necessary for mypy\n                    value = self._deserialize_pytorch(value, recursion_depth=depth_pp)\n                    step_name = value['step_name']\n                    key = value['key']\n                    if key not in components:\n                        key = str(key)\n                    component = self._deserialize_pytorch(\n                        components[key],\n                        initialize_with_defaults=initialize_with_defaults,\n                        recursion_depth=depth_pp\n                    )\n                    # The component is now added to where it should be used\n                    # later. It should not be passed to the constructor of the\n                    # main flow object.\n                    del components[key]\n                    if step_name is None:\n                        rval = component\n                    elif 'argument_1' not in value:\n                        rval = (step_name, component)\n                    else:\n                        rval = (step_name, component, value['argument_1'])\n                else:\n                    raise ValueError('Cannot flow_to_pytorch %s' % serialized_type)\n\n            else:\n                rval = OrderedDict(\n                    (\n                        self._deserialize_pytorch(\n                            o=key,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                        ),\n                        self._deserialize_pytorch(\n                            o=value,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                        )\n                    )\n                    for key, value in sorted(o.items())\n                )\n        elif isinstance(o, (list, tuple)):\n            rval = [\n                self._deserialize_pytorch(\n                    o=element,\n                    components=components,\n                    initialize_with_defaults=initialize_with_defaults,\n                    recursion_depth=depth_pp,\n                )\n                for element in o\n            ]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, (bool, int, float, str)) or o is None:\n            rval = o\n        elif isinstance(o, OpenMLFlow):\n            if not self._is_pytorch_flow(o):\n                raise ValueError('Only pytorch flows can be reinstantiated')\n            rval = self._deserialize_model(\n                flow=o,\n                keep_defaults=initialize_with_defaults,\n                recursion_depth=recursion_depth,\n            )\n        else:\n            raise TypeError(o)\n        logging.info('-%s flow_to_pytorch END   o=%s, rval=%s'\n                     % ('-' * recursion_depth, o, rval))\n        return rval\n\n    def model_to_flow(self, model: Any, custom_name: Optional[str] = None) -&gt; 'OpenMLFlow':\n        \"\"\"Transform a Pytorch model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n        # Necessary to make pypy not complain about all the different possible return types\n        return self._serialize_pytorch(model, custom_name)\n\n    def _serialize_pytorch(self, o: Any, parent_model: Optional[Any] = None, custom_name: Optional[str] = None) -&gt; Any:\n        rval = None  # type: Any\n        if self.is_estimator(o):\n            # is the main model or a submodel\n            rval = self._serialize_model(o, custom_name)\n        elif isinstance(o, (list, tuple)):\n            rval = [self._serialize_pytorch(element, parent_model) for element in o]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, SIMPLE_TYPES) or o is None:\n            if isinstance(o, tuple(SIMPLE_NUMPY_TYPES)):\n                o = o.item()\n            # base parameter values\n            rval = o\n        elif isinstance(o, dict):\n            if not isinstance(o, OrderedDict):\n                o = OrderedDict([(key, value) for key, value in sorted(o.items())])\n\n            rval = OrderedDict()\n            for key, value in o.items():\n                if not isinstance(key, str):\n                    raise TypeError('Can only use string as keys, you passed '\n                                    'type %s for value %s.' %\n                                    (type(key), str(key)))\n                key = self._serialize_pytorch(key, parent_model)\n                value = self._serialize_pytorch(value, parent_model)\n                rval[key] = value\n            rval = rval\n        elif isinstance(o, type):\n            rval = self._serialize_type(o)\n        # This only works for user-defined functions (and not even partial).\n        # I think this is exactly what we want here as there shouldn't be any\n        # built-in or functool.partials in a pipeline\n        elif inspect.isfunction(o):\n            rval = self._serialize_function(o)\n        elif inspect.ismethoddescriptor(o):\n            rval = self._serialize_methoddescriptor(o)\n        else:\n            raise TypeError(o, type(o))\n        return rval\n\n    def get_version_information(self) -&gt; List[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Libraries listed are ``Python``, ``pytorch``, ``numpy`` and ``scipy``.\n\n        Returns\n        -------\n        List\n        \"\"\"\n\n        # This can possibly be done by a package such as pyxb, but I could not get\n        # it to work properly.\n        import scipy\n        import numpy\n\n        major, minor, micro, _, _ = sys.version_info\n        python_version = 'Python_{}.'.format(\n            \".\".join([str(major), str(minor), str(micro)]))\n        pytorch_version = 'Torch_{}.'.format(torch.__version__)\n        numpy_version = 'NumPy_{}.'.format(numpy.__version__)\n        scipy_version = 'SciPy_{}.'.format(scipy.__version__)\n        pytorch_version_formatted = pytorch_version.replace('+','_')\n        return [python_version, pytorch_version_formatted, numpy_version, scipy_version]\n\n    def create_setup_string(self, model: Any) -&gt; str:\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n        run_environment = \" \".join(self.get_version_information())\n        return run_environment + \" \" + str(model)\n\n    @classmethod\n    def _is_pytorch_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        return (\n            flow.external_version.startswith('torch==')\n            or ',torch==' in flow.external_version\n        )\n\n    def _serialize_model(self, model: Any, custom_name: Optional[str] = None) -&gt; OpenMLFlow:\n        \"\"\"Create an OpenMLFlow.\n\n        Calls `pytorch_to_flow` recursively to properly serialize the\n        parameters to strings and the components (other models) to OpenMLFlows.\n\n        Parameters\n        ----------\n        model : pytorch estimator\n\n        Returns\n        -------\n        OpenMLFlow\n\n        \"\"\"\n\n        # Get all necessary information about the model objects itself\n        parameters, parameters_meta_info, subcomponents, subcomponents_explicit = \\\n            self._extract_information_from_model(model)\n\n        # Check that a component does not occur multiple times in a flow as this\n        # is not supported by OpenML\n        self._check_multiple_occurence_of_component_in_flow(model, subcomponents)\n\n        import zlib\n        import os\n\n        # class_name = model.__module__ + \".\" + model.__class__.__name__\n        class_name = 'torch.nn' + \".\" + model.__class__.__name__\n        class_name += '.'\n        class_name += format(zlib.crc32(bytearray(os.urandom(32))), 'x')\n        class_name += format(zlib.crc32(bytearray(os.urandom(32))), 'x')\n\n        name = class_name\n\n        # Get the external versions of all sub-components\n        external_version = self._get_external_version_string(model, subcomponents)\n\n        dependencies = '\\n'.join([\n            self._format_external_version(\n                'torch',\n                torch.__version__,\n            ),\n            'numpy&gt;=1.6.1',\n            'scipy&gt;=0.9',\n        ])\n\n        torch_version = self._format_external_version('torch', torch.__version__)\n        torch_version_formatted = torch_version.replace('==', '_')\n        torch_version_formatted = torch_version_formatted.replace('+', '_')\n\n        flow = OpenMLFlow(name=name,\n                          class_name=class_name,\n                          description='Automatically created pytorch flow.',\n                          model=model,\n                          components=subcomponents,\n                          parameters=parameters,\n                          parameters_meta_info=parameters_meta_info,\n                          external_version=external_version,\n                          tags=['openml-python', 'pytorch',\n                                'python', torch_version_formatted],\n                          language='English',\n                          dependencies=dependencies, \n                          custom_name=custom_name)\n\n        return flow\n\n    def _get_external_version_string(\n        self,\n        model: Any,\n        sub_components: Dict[str, OpenMLFlow],\n    ) -&gt; str:\n        # Create external version string for a flow, given the model and the\n        # already parsed dictionary of sub_components. Retrieves the external\n        # version of all subcomponents, which themselves already contain all\n        # requirements for their subcomponents. The external version string is a\n        # sorted concatenation of all modules which are present in this run.\n        model_package_name = model.__module__.split('.')[0]\n        module = importlib.import_module(model_package_name)\n        model_package_version_number = 'module.__version__'  # type: ignore\n        external_version = self._format_external_version(\n            model_package_name, model_package_version_number,\n        )\n        openml_version = self._format_external_version('openml', openml.__version__)\n        torch_version = self._format_external_version('torch', torch.__version__)\n        external_versions = set()\n        external_versions.add(external_version)\n        external_versions.add(openml_version)\n        external_versions.add(torch_version)\n        for visitee in sub_components.values():\n            for external_version in visitee.external_version.split(','):\n                external_versions.add(external_version)\n        return ','.join(list(sorted(external_versions)))\n\n    def _check_multiple_occurence_of_component_in_flow(\n        self,\n        model: Any,\n        sub_components: Dict[str, OpenMLFlow],\n    ) -&gt; None:\n        to_visit_stack = []  # type: List[OpenMLFlow]\n        to_visit_stack.extend(sub_components.values())\n        known_sub_components = set()  # type: Set[str]\n        while len(to_visit_stack) &gt; 0:\n            visitee = to_visit_stack.pop()\n            if visitee.name in known_sub_components:\n                raise ValueError('Found a second occurence of component %s when '\n                                 'trying to serialize %s.' % (visitee.name, model))\n            else:\n                known_sub_components.add(visitee.name)\n                to_visit_stack.extend(visitee.components.values())\n\n    def _is_container_module(self, module: torch.nn.Module) -&gt; bool:\n        if isinstance(module,\n                      (torch.nn.Sequential,\n                       torch.nn.ModuleDict,\n                       torch.nn.ModuleList)):\n            return True\n        if module in (torch.nn.modules.container.Sequential,\n                      torch.nn.modules.container.ModuleDict,\n                      torch.nn.modules.container.ModuleList):\n            return True\n        return False\n\n    def _get_module_hyperparameters(self, module: torch.nn.Module,\n                                    parameters: Dict[str, torch.nn.Parameter]) -&gt; Dict[str, Any]:\n        # Extract the signature of the module constructor\n        main_signature = inspect.signature(module.__init__)\n        params = dict()  # type: Dict[str, Any]\n\n        check_bases = False  # type: bool\n        for param_name, param in main_signature.parameters.items():\n            # Skip hyper-parameters which are actually parameters.\n            if param_name in parameters.keys():\n                continue\n\n            # Skip *args and **kwargs, and check the base classes instead.\n            if param.kind in (inspect.Parameter.VAR_POSITIONAL,\n                              inspect.Parameter.VAR_KEYWORD):\n                check_bases = True\n                continue\n\n            # Extract the hyperparameter from the module.\n            if hasattr(module, param_name):\n                params[param_name] = getattr(module, param_name)\n\n        if check_bases:\n            for base in module.__class__.__bases__:\n                # Extract the signature  of the base constructor\n                base_signature = inspect.signature(base.__init__)\n\n                for param_name, param in base_signature.parameters.items():\n                    # Skip hyper-parameters which are actually parameters.\n                    if param_name in parameters.keys():\n                        continue\n\n                    # Skip *args and **kwargs since they are not relevant.\n                    if param.kind in (inspect.Parameter.VAR_POSITIONAL,\n                                      inspect.Parameter.VAR_KEYWORD):\n                        continue\n\n                    # Extract the hyperparameter from the module.\n                    if hasattr(module, param_name):\n                        params[param_name] = getattr(module, param_name)\n\n        from .layers import Functional\n        if isinstance(module, Functional):\n            params['args'] = getattr(module, 'args')\n            params['kwargs'] = getattr(module, 'kwargs')\n\n        return params\n\n    def _get_module_descriptors(self, model: torch.nn.Module, deep=True) -&gt; Dict[str, Any]:\n        # The named children (modules) of the given module.\n        named_children = list((k, v) for (k, v) in model.named_children())\n        # The parameters of the given module and its submodules.\n        model_parameters = dict((k, v) for (k, v) in model.named_parameters())\n\n        parameters = dict()  # type: Dict[str, Any]\n\n        if not self._is_container_module(model):\n            # For non-containers, we simply extract the hyperparameters.\n            parameters = self._get_module_hyperparameters(model, model_parameters)\n        else:\n            # Otherwise we serialize their children as lists of pairs in order\n            # to maintain the order of the sub modules.\n            parameters['children'] = named_children\n\n        # If a deep description is required, append the children to the dictionary of\n        # returned parameters.\n        if deep:\n            named_children_dict = dict(named_children)\n            parameters = {**parameters, **named_children_dict}\n\n        return parameters\n\n    def _extract_information_from_model(\n        self,\n        model: Any,\n    ) -&gt; Tuple[\n        'OrderedDict[str, Optional[str]]',\n        'OrderedDict[str, Optional[Dict]]',\n        'OrderedDict[str, OpenMLFlow]',\n        Set,\n    ]:\n        # This function contains four \"global\" states and is quite long and\n        # complicated. If it gets to complicated to ensure it's correctness,\n        # it would be best to make it a class with the four \"global\" states being\n        # the class attributes and the if/elif/else in the for-loop calls to\n        # separate class methods\n\n        # stores all entities that should become subcomponents\n        sub_components = OrderedDict()  # type: OrderedDict[str, OpenMLFlow]\n        # stores the keys of all subcomponents that should become\n        sub_components_explicit = set()\n        parameters = OrderedDict()  # type: OrderedDict[str, Optional[str]]\n        parameters_meta_info = OrderedDict()  # type: OrderedDict[str, Optional[Dict]]\n\n        model_parameters = self._get_module_descriptors(model, deep=True)\n        for k, v in sorted(model_parameters.items(), key=lambda t: t[0]):\n            rval = self._serialize_pytorch(v, model)\n\n            def flatten_all(list_):\n                \"\"\" Flattens arbitrary depth lists of lists (e.g. [[1,2],[3,[1]]] -&gt; [1,2,3,1]). \"\"\"\n                for el in list_:\n                    if isinstance(el, (list, tuple)):\n                        yield from flatten_all(el)\n                    else:\n                        yield el\n\n            is_non_empty_list_of_lists_with_same_type = (\n                isinstance(rval, (list, tuple))\n                and len(rval) &gt; 0\n                and isinstance(rval[0], (list, tuple))\n                and all([isinstance(rval_i, type(rval[0])) for rval_i in rval])\n            )\n\n            # Check that all list elements are of simple types.\n            nested_list_of_simple_types = (\n                is_non_empty_list_of_lists_with_same_type\n                and all([isinstance(el, SIMPLE_TYPES) for el in flatten_all(rval)])\n            )\n\n            if is_non_empty_list_of_lists_with_same_type and not nested_list_of_simple_types:\n                # If a list of lists is identified that include 'non-simple' types (e.g. objects),\n                # we assume they are steps in a pipeline, feature union, or base classifiers in\n                # a voting classifier.\n                parameter_value = list()  # type: List\n                reserved_keywords = set(self._get_module_descriptors(model, deep=False).keys())\n\n                for sub_component_tuple in rval:\n                    identifier = sub_component_tuple[0]\n                    sub_component = sub_component_tuple[1]\n                    sub_component_type = type(sub_component_tuple)\n                    if not 2 &lt;= len(sub_component_tuple) &lt;= 3:\n                        msg = 'Length of tuple does not match assumptions'\n                        raise ValueError(msg)\n                    if not isinstance(sub_component, (OpenMLFlow, type(None))):\n                        msg = 'Second item of tuple does not match assumptions. ' \\\n                              'Expected OpenMLFlow, got %s' % type(sub_component)\n                        raise TypeError(msg)\n\n                    if identifier in reserved_keywords:\n                        parent_model = \"{}.{}\".format(model.__module__,\n                                                      model.__class__.__name__)\n                        msg = 'Found element shadowing official ' \\\n                              'parameter for %s: %s' % (parent_model,\n                                                        identifier)\n                        raise PyOpenMLError(msg)\n\n                    if sub_component is None:\n                        # In a FeatureUnion it is legal to have a None step\n\n                        pv = [identifier, None]\n                        if sub_component_type is tuple:\n                            parameter_value.append(tuple(pv))\n                        else:\n                            parameter_value.append(pv)\n\n                    else:\n                        # Add the component to the list of components, add a\n                        # component reference as a placeholder to the list of\n                        # parameters, which will be replaced by the real component\n                        # when deserializing the parameter\n                        sub_components_explicit.add(identifier)\n                        sub_components[identifier] = sub_component\n                        component_reference = OrderedDict()  # type: Dict[str, Union[str, Dict]]\n                        component_reference['oml-python:serialized_object'] = 'component_reference'\n                        cr_value = OrderedDict()  # type: Dict[str, Any]\n                        cr_value['key'] = identifier\n                        cr_value['step_name'] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value['argument_1'] = sub_component_tuple[2]\n                        component_reference['value'] = cr_value\n                        parameter_value.append(component_reference)\n\n                # Here (and in the elif and else branch below) are the only\n                # places where we encode a value as json to make sure that all\n                # parameter values still have the same type after\n                # deserialization\n\n                if isinstance(rval, tuple):\n                    parameter_json = json.dumps(tuple(parameter_value))\n                else:\n                    parameter_json = json.dumps(parameter_value)\n                parameters[k] = parameter_json\n\n            elif isinstance(rval, OpenMLFlow):\n\n                # A subcomponent, for example the layers in a sequential model\n                sub_components[k] = rval\n                sub_components_explicit.add(k)\n                component_reference = OrderedDict()\n                component_reference['oml-python:serialized_object'] = 'component_reference'\n                cr_value = OrderedDict()\n                cr_value['key'] = k\n                cr_value['step_name'] = None\n                component_reference['value'] = cr_value\n                cr = self._serialize_pytorch(component_reference, model)\n                parameters[k] = json.dumps(cr)\n\n            else:\n                # a regular hyperparameter\n                rval = json.dumps(rval)\n                parameters[k] = rval\n\n            parameters_meta_info[k] = OrderedDict((('description', None), ('data_type', None)))\n\n        return parameters, parameters_meta_info, sub_components, sub_components_explicit\n\n    def _get_fn_arguments_with_defaults(self, fn_name: Callable) -&gt; Tuple[Dict, Set]:\n        \"\"\"\n        Returns:\n            i) a dict with all parameter names that have a default value, and\n            ii) a set with all parameter names that do not have a default\n\n        Parameters\n        ----------\n        fn_name : callable\n            The function of which we want to obtain the defaults\n\n        Returns\n        -------\n        params_with_defaults: dict\n            a dict mapping parameter name to the default value\n        params_without_defaults: set\n            a set with all parameters that do not have a default value\n        \"\"\"\n        # parameters with defaults are optional, all others are required.\n        signature = inspect.getfullargspec(fn_name)\n        if signature.defaults:\n            optional_params = dict(zip(reversed(signature.args), reversed(signature.defaults)))\n        else:\n            optional_params = dict()\n        required_params = {arg for arg in signature.args if arg not in optional_params}\n        return optional_params, required_params\n\n    def _deserialize_model(\n        self,\n        flow: OpenMLFlow,\n        keep_defaults: bool,\n        recursion_depth: int,\n    ) -&gt; Any:\n        logging.info('-%s deserialize %s' % ('-' * recursion_depth, flow.name))\n        model_name = flow.class_name\n        self._check_dependencies(flow.dependencies)\n\n        parameters = flow.parameters\n        components = flow.components\n        parameter_dict = OrderedDict()  # type: Dict[str, Any]\n\n        # Do a shallow copy of the components dictionary so we can remove the\n        # components from this copy once we added them into the pipeline. This\n        # allows us to not consider them any more when looping over the\n        # components, but keeping the dictionary of components untouched in the\n        # original components dictionary.\n        components_ = copy.copy(components)\n\n        for name in parameters:\n            value = parameters.get(name)\n            logging.info('--%s flow_parameter=%s, value=%s' %\n                         ('-' * recursion_depth, name, value))\n            rval = self._deserialize_pytorch(\n                value,\n                components=components_,\n                initialize_with_defaults=keep_defaults,\n                recursion_depth=recursion_depth + 1,\n            )\n            parameter_dict[name] = rval\n\n        for name in components:\n            if name in parameter_dict:\n                continue\n            if name not in components_:\n                continue\n            value = components[name]\n            logging.info('--%s flow_component=%s, value=%s'\n                         % ('-' * recursion_depth, name, value))\n            rval = self._deserialize_pytorch(\n                value,\n                recursion_depth=recursion_depth + 1,\n            )\n            parameter_dict[name] = rval\n\n        # Remove the unique identifier\n        model_name = model_name.rsplit('.', 1)[0]\n\n        module_name = model_name.rsplit('.', 1)\n        model_class = getattr(importlib.import_module(module_name[0]),\n                              module_name[1])\n\n        if keep_defaults:\n            # obtain all params with a default\n            param_defaults, _ = \\\n                self._get_fn_arguments_with_defaults(model_class.__init__)\n\n            # delete the params that have a default from the dict,\n            # so they get initialized with their default value\n            # except [...]\n            for param in param_defaults:\n                # [...] the ones that also have a key in the components dict.\n                # As OpenML stores different flows for ensembles with different\n                # (base-)components, in OpenML terms, these are not considered\n                # hyperparameters but rather constants (i.e., changing them would\n                # result in a different flow)\n                if param not in components.keys() and param in parameter_dict:\n                    del parameter_dict[param]\n\n        if self._is_container_module(model_class):\n            children = parameter_dict['children']\n            children = list((str(k), v) for (k, v) in children)\n            children = OrderedDict(children)\n            return model_class(children)\n\n        from .layers import Functional\n        if model_class is Functional:\n            return model_class(function=parameter_dict['function'],\n                               *parameter_dict['args'],\n                               **parameter_dict['kwargs'])\n\n        return model_class(**parameter_dict)\n\n    def _check_dependencies(self, dependencies: str) -&gt; None:\n        if not dependencies:\n            return\n\n        dependencies_list = dependencies.split('\\n')\n        for dependency_string in dependencies_list:\n            match = DEPENDENCIES_PATTERN.match(dependency_string)\n            if not match:\n                raise ValueError('Cannot parse dependency %s' % dependency_string)\n\n            dependency_name = match.group('name')\n            operation = match.group('operation')\n            version = match.group('version')\n\n            module = importlib.import_module(dependency_name)\n            required_version = LooseVersion(version)\n            installed_version = LooseVersion(module.__version__)  # type: ignore\n\n            if operation == '==':\n                check = required_version == installed_version\n            elif operation == '&gt;':\n                check = installed_version &gt; required_version\n            elif operation == '&gt;=':\n                check = (installed_version &gt; required_version\n                         or installed_version == required_version)\n            else:\n                raise NotImplementedError(\n                    'operation \\'%s\\' is not supported' % operation)\n            if not check:\n                raise ValueError('Trying to deserialize a model with dependency '\n                                 '%s not satisfied.' % dependency_string)\n\n    def _serialize_type(self, o: Any) -&gt; 'OrderedDict[str, str]':\n        mapping = {float: 'float',\n                   np.float: 'np.float',\n                   np.float32: 'np.float32',\n                   np.float64: 'np.float64',\n                   int: 'int',\n                   np.int: 'np.int',\n                   np.int32: 'np.int32',\n                   np.int64: 'np.int64'}\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret['oml-python:serialized_object'] = 'type'\n        ret['value'] = mapping[o]\n        return ret\n\n    def _deserialize_type(self, o: str) -&gt; Any:\n        mapping = {'float': float,\n                   'np.float': np.float,\n                   'np.float32': np.float32,\n                   'np.float64': np.float64,\n                   'int': int,\n                   'np.int': np.int,\n                   'np.int32': np.int32,\n                   'np.int64': np.int64}\n        return mapping[o]\n\n    def _serialize_function(self, o: Callable) -&gt; 'OrderedDict[str, str]':\n        name = o.__module__ + '.' + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret['oml-python:serialized_object'] = 'function'\n        ret['value'] = name\n        return ret\n\n    def _deserialize_function(self, name: str) -&gt; Callable:\n        module_name = name.rsplit('.', 1)\n        function_handle = getattr(importlib.import_module(module_name[0]), module_name[1])\n        return function_handle\n\n    def _serialize_methoddescriptor(self, o: Any) -&gt; 'OrderedDict[str, str]':\n        name = o.__objclass__.__module__ \\\n            + '.' + o.__objclass__.__name__ \\\n            + '.' + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret['oml-python:serialized_object'] = 'methoddescriptor'\n        ret['value'] = name\n        return ret\n\n    def _deserialize_methoddescriptor(self, name: str) -&gt; Any:\n        module_name = name.rsplit('.', 2)\n        object_handle = getattr(importlib.import_module(module_name[0]), module_name[1])\n        function_handle = getattr(object_handle, module_name[2])\n        return function_handle\n\n    def _format_external_version(\n        self,\n        model_package_name: str,\n        model_package_version_number: str,\n    ) -&gt; str:\n        return '%s==%s' % (model_package_name, model_package_version_number)\n\n    @staticmethod\n    def _get_parameter_values_recursive(param_grid: Union[Dict, List[Dict]],\n                                        parameter_name: str) -&gt; List[Any]:\n        \"\"\"\n        Returns a list of values for a given hyperparameter, encountered\n        recursively throughout the flow. (e.g., n_jobs can be defined\n        for various flows)\n\n        Parameters\n        ----------\n        param_grid: Union[Dict, List[Dict]]\n            Dict mapping from hyperparameter list to value, to a list of\n            such dicts\n\n        parameter_name: str\n            The hyperparameter that needs to be inspected\n\n        Returns\n        -------\n        List\n            A list of all values of hyperparameters with this name\n        \"\"\"\n        if isinstance(param_grid, dict):\n            result = list()\n            for param, value in param_grid.items():\n                if param.split('__')[-1] == parameter_name:\n                    result.append(value)\n            return result\n        elif isinstance(param_grid, list):\n            result = list()\n            for sub_grid in param_grid:\n                result.extend(PytorchExtension._get_parameter_values_recursive(sub_grid,\n                                                                               parameter_name))\n            return result\n        else:\n            raise ValueError('Param_grid should either be a dict or list of dicts')\n\n    ################################################################################################\n    # Methods for performing runs with extension modules\n\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is a pytorch estimator.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, torch.nn.Module)\n\n    def seed_model(self, model: Any, seed: Optional[int] = None) -&gt; Any:\n        \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n        model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Models that are already seeded will maintain the seed. In this case,\n        only integer seeds are allowed (An exception is raised when a RandomState was used as\n        seed).\n\n        Parameters\n        ----------\n        model : pytorch model\n            The model to be seeded\n        seed : int\n            The seed to initialize the RandomState with. Unseeded subcomponents\n            will be seeded with a random number from the RandomState.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        return model\n\n    def _run_model_on_fold(\n        self,\n        model: Any,\n        task: 'OpenMLTask',\n        X_train: Union[np.ndarray, scipy.sparse.spmatrix, pd.DataFrame],\n        rep_no: int,\n        fold_no: int,\n        y_train: Optional[np.ndarray] = None,\n        X_test: Optional[Union[np.ndarray, scipy.sparse.spmatrix, pd.DataFrame]] = None,\n    ) -&gt; Tuple[\n        np.ndarray,\n        np.ndarray,\n        'OrderedDict[str, float]',\n        Optional[OpenMLRunTrace],\n        Optional[Any]\n    ]:\n        \"\"\"Run a model on a repeat,fold,subsample triplet of the task and return prediction\n        information.\n\n        Furthermore, it will measure run time measures in case multi-core behaviour allows this.\n        * exact user cpu time will be measured if the number of cores is set (recursive throughout\n        the model) exactly to 1\n        * wall clock time will be measured if the number of cores is set (recursive throughout the\n        model) to any given number (but not when it is set to -1)\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        run_task_get_arff_content. Do not use this function unless you know what you are doing.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        predictions : np.ndarray\n            Model predictions.\n        probabilities :  Optional, np.ndarray\n            Predicted probabilities (only applicable for supervised classification tasks).\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : Optional, OpenMLRunTrace\n            Hyperparameter optimization trace (only applicable for supervised tasks with\n            hyperparameter optimization).\n        additional_information: Optional, Any\n            Additional information provided by the extension to be converted into additional files.\n        \"\"\"\n\n        try:\n            trainer:OpenMLTrainerModule = config.trainer\n            trainer.logger = config.logger\n        except AttributeError:\n            raise ValueError('Trainer not set to config. Please use openml_pytorch.config.trainer = trainer to set the trainer.')\n        return trainer.run_model_on_fold(model, task, X_train, rep_no, fold_no, y_train, X_test)\n\n\n    def compile_additional_information(\n            self,\n            task: 'OpenMLTask',\n            additional_information: List[Tuple[int, int, Any]]\n    ) -&gt; Dict[str, Tuple[str, str]]:\n        \"\"\"Compiles additional information provided by the extension during the runs into a final\n        set of files.\n\n        Parameters\n        ----------\n        task : OpenMLTask\n            The task the model was run on.\n        additional_information: List[Tuple[int, int, Any]]\n            A list of (fold, repetition, additional information) tuples obtained during training.\n\n        Returns\n        -------\n        files : Dict[str, Tuple[str, str]]\n            A dictionary of files with their file name and contents.\n        \"\"\"\n        return dict()\n\n    def obtain_parameter_values(\n        self,\n        flow: 'OpenMLFlow',\n        model: Any = None,\n    ) -&gt; List[Dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n        openml.flows.functions._check_flow_for_server_id(flow)\n\n        def get_flow_dict(_flow):\n            flow_map = {_flow.name: _flow.flow_id}\n            for subflow in _flow.components:\n                flow_map.update(get_flow_dict(_flow.components[subflow]))\n            return flow_map\n\n        def extract_parameters(_flow, _flow_dict, component_model,\n                               _main_call=False, main_id=None):\n            def is_subcomponent_specification(values):\n                # checks whether the current value can be a specification of\n                # subcomponents, as for example the value for steps parameter\n                # (in Pipeline) or transformers parameter (in\n                # ColumnTransformer). These are always lists/tuples of lists/\n                # tuples, size bigger than 2 and an OpenMLFlow item involved.\n                if not isinstance(values, (tuple, list)):\n                    return False\n                for item in values:\n                    if not isinstance(item, (tuple, list)):\n                        return False\n                    if len(item) &lt; 2:\n                        return False\n                    if not isinstance(item[1], openml.flows.OpenMLFlow):\n                        return False\n                return True\n\n            # _flow is openml flow object, _param dict maps from flow name to flow\n            # id for the main call, the param dict can be overridden (useful for\n            # unit tests / sentinels) this way, for flows without subflows we do\n            # not have to rely on _flow_dict\n            exp_parameters = set(_flow.parameters)\n            exp_components = set(_flow.components)\n            model_parameters = set([mp for mp in self._get_module_descriptors(component_model)\n                                    if '__' not in mp])\n            if len((exp_parameters | exp_components) ^ model_parameters) != 0:\n                flow_params = sorted(exp_parameters | exp_components)\n                model_params = sorted(model_parameters)\n                raise ValueError('Parameters of the model do not match the '\n                                 'parameters expected by the '\n                                 'flow:\\nexpected flow parameters: '\n                                 '%s\\nmodel parameters: %s' % (flow_params,\n                                                               model_params))\n\n            _params = []\n            for _param_name in _flow.parameters:\n                _current = OrderedDict()\n                _current['oml:name'] = _param_name\n\n                current_param_values = self.model_to_flow(\n                    self._get_module_descriptors(component_model)[_param_name])\n\n                # Try to filter out components (a.k.a. subflows) which are\n                # handled further down in the code (by recursively calling\n                # this function)!\n                if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                    continue\n\n                if is_subcomponent_specification(current_param_values):\n                    # complex parameter value, with subcomponents\n                    parsed_values = list()\n                    for subcomponent in current_param_values:\n                        if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                            raise ValueError('Component reference should be '\n                                             'size {2,3}. ')\n\n                        subcomponent_identifier = subcomponent[0]\n                        subcomponent_flow = subcomponent[1]\n                        if not isinstance(subcomponent_identifier, str):\n                            raise TypeError('Subcomponent identifier should be '\n                                            'string')\n                        if not isinstance(subcomponent_flow,\n                                          openml.flows.OpenMLFlow):\n                            raise TypeError('Subcomponent flow should be string')\n\n                        current = {\n                            \"oml-python:serialized_object\": \"component_reference\",\n                            \"value\": {\n                                \"key\": subcomponent_identifier,\n                                \"step_name\": subcomponent_identifier\n                            }\n                        }\n                        if len(subcomponent) == 3:\n                            if not isinstance(subcomponent[2], list):\n                                raise TypeError('Subcomponent argument should be'\n                                                'list')\n                            current['value']['argument_1'] = subcomponent[2]\n                        parsed_values.append(current)\n                    parsed_values = json.dumps(parsed_values)\n                else:\n                    # vanilla parameter value\n                    parsed_values = json.dumps(current_param_values)\n\n                _current['oml:value'] = parsed_values\n                if _main_call:\n                    _current['oml:component'] = main_id\n                else:\n                    _current['oml:component'] = _flow_dict[_flow.name]\n                _params.append(_current)\n\n            for _identifier in _flow.components:\n                subcomponent_model = self._get_module_descriptors(component_model)[_identifier]\n                _params.extend(extract_parameters(_flow.components[_identifier],\n                                                  _flow_dict, subcomponent_model))\n            return _params\n\n        flow_dict = get_flow_dict(flow)\n        model = model if model is not None else flow.model\n        parameters = extract_parameters(flow, flow_dict, model, True, flow.flow_id)\n\n        return parameters\n\n    def _openml_param_name_to_pytorch(\n        self,\n        openml_parameter: openml.setups.OpenMLParameter,\n        flow: OpenMLFlow,\n    ) -&gt; str:\n        \"\"\"\n        Converts the name of an OpenMLParameter into the pytorch name, given a flow.\n\n        Parameters\n        ----------\n        openml_parameter: OpenMLParameter\n            The parameter under consideration\n\n        flow: OpenMLFlow\n            The flow that provides context.\n\n        Returns\n        -------\n        pytorch_parameter_name: str\n            The name the parameter will have once used in pytorch\n        \"\"\"\n        if not isinstance(openml_parameter, openml.setups.OpenMLParameter):\n            raise ValueError('openml_parameter should be an instance of OpenMLParameter')\n        if not isinstance(flow, OpenMLFlow):\n            raise ValueError('flow should be an instance of OpenMLFlow')\n\n        flow_structure = flow.get_structure('name')\n        if openml_parameter.flow_name not in flow_structure:\n            raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not correspond. ')\n        name = openml_parameter.flow_name  # for PEP8\n        return '__'.join(flow_structure[name] + [openml_parameter.parameter_name])\n\n    ################################################################################################\n    # Methods for hyperparameter optimization\n\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n        optimization model (UNUSED)\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        return model\n\n\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained\n        Parameters\n        ----------\n        model : Any\n        Returns\n        -------\n        bool\n        \"\"\"\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>classmethod</code>","text":"<p>Check whether a given describes a Pytorch estimator.</p> <p>This is done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>@classmethod\ndef can_handle_flow(cls, flow: 'OpenMLFlow') -&gt; bool:\n    \"\"\"Check whether a given describes a Pytorch estimator.\n\n    This is done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return cls._is_pytorch_flow(flow)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>classmethod</code>","text":"<p>Check whether a model is an instance of <code>torch.nn.Module</code>.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>@classmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model is an instance of ``torch.nn.Module``.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    from torch.nn import Module\n    return isinstance(model, Module)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained\n    Parameters\n    ----------\n    model : Any\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.compile_additional_information","title":"<code>compile_additional_information(task, additional_information)</code>","text":"<p>Compiles additional information provided by the extension during the runs into a final set of files.</p> <p>Parameters:</p> Name Type Description Default <code>task</code> <code>OpenMLTask</code> <p>The task the model was run on.</p> required <code>additional_information</code> <code>List[Tuple[int, int, Any]]</code> <p>A list of (fold, repetition, additional information) tuples obtained during training.</p> required <p>Returns:</p> Name Type Description <code>files</code> <code>Dict[str, Tuple[str, str]]</code> <p>A dictionary of files with their file name and contents.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def compile_additional_information(\n        self,\n        task: 'OpenMLTask',\n        additional_information: List[Tuple[int, int, Any]]\n) -&gt; Dict[str, Tuple[str, str]]:\n    \"\"\"Compiles additional information provided by the extension during the runs into a final\n    set of files.\n\n    Parameters\n    ----------\n    task : OpenMLTask\n        The task the model was run on.\n    additional_information: List[Tuple[int, int, Any]]\n        A list of (fold, repetition, additional information) tuples obtained during training.\n\n    Returns\n    -------\n    files : Dict[str, Tuple[str, str]]\n        A dictionary of files with their file name and contents.\n    \"\"\"\n    return dict()\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.create_setup_string","title":"<code>create_setup_string(model)</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def create_setup_string(self, model: Any) -&gt; str:\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n    run_environment = \" \".join(self.get_version_information())\n    return run_environment + \" \" + str(model)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False)</code>","text":"<p>Initializes a Pytorch model based on a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>mixed</code> <p>the object to deserialize (can be flow object, or any serialized parameter value that is accepted by)</p> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <p>Returns:</p> Type Description <code>mixed</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def flow_to_model(self, flow: 'OpenMLFlow', initialize_with_defaults: bool = False) -&gt; Any:\n    \"\"\"Initializes a Pytorch model based on a flow.\n\n    Parameters\n    ----------\n    flow : mixed\n        the object to deserialize (can be flow object, or any serialized\n        parameter value that is accepted by)\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    Returns\n    -------\n    mixed\n    \"\"\"\n    return self._deserialize_pytorch(flow, initialize_with_defaults=initialize_with_defaults)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.get_version_information","title":"<code>get_version_information()</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Libraries listed are <code>Python</code>, <code>pytorch</code>, <code>numpy</code> and <code>scipy</code>.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def get_version_information(self) -&gt; List[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Libraries listed are ``Python``, ``pytorch``, ``numpy`` and ``scipy``.\n\n    Returns\n    -------\n    List\n    \"\"\"\n\n    # This can possibly be done by a package such as pyxb, but I could not get\n    # it to work properly.\n    import scipy\n    import numpy\n\n    major, minor, micro, _, _ = sys.version_info\n    python_version = 'Python_{}.'.format(\n        \".\".join([str(major), str(minor), str(micro)]))\n    pytorch_version = 'Torch_{}.'.format(torch.__version__)\n    numpy_version = 'NumPy_{}.'.format(numpy.__version__)\n    scipy_version = 'SciPy_{}.'.format(scipy.__version__)\n    pytorch_version_formatted = pytorch_version.replace('+','_')\n    return [python_version, pytorch_version_formatted, numpy_version, scipy_version]\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>","text":"<p>Instantiate a <code>base_estimator</code> which can be searched over by the hyperparameter optimization model (UNUSED)</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n    optimization model (UNUSED)\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    return model\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.is_estimator","title":"<code>is_estimator(model)</code>","text":"<p>Check whether the given model is a pytorch estimator.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is a pytorch estimator.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return isinstance(model, torch.nn.Module)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.model_to_flow","title":"<code>model_to_flow(model, custom_name=None)</code>","text":"<p>Transform a Pytorch model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def model_to_flow(self, model: Any, custom_name: Optional[str] = None) -&gt; 'OpenMLFlow':\n    \"\"\"Transform a Pytorch model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n    # Necessary to make pypy not complain about all the different possible return types\n    return self._serialize_pytorch(model, custom_name)\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def obtain_parameter_values(\n    self,\n    flow: 'OpenMLFlow',\n    model: Any = None,\n) -&gt; List[Dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n    openml.flows.functions._check_flow_for_server_id(flow)\n\n    def get_flow_dict(_flow):\n        flow_map = {_flow.name: _flow.flow_id}\n        for subflow in _flow.components:\n            flow_map.update(get_flow_dict(_flow.components[subflow]))\n        return flow_map\n\n    def extract_parameters(_flow, _flow_dict, component_model,\n                           _main_call=False, main_id=None):\n        def is_subcomponent_specification(values):\n            # checks whether the current value can be a specification of\n            # subcomponents, as for example the value for steps parameter\n            # (in Pipeline) or transformers parameter (in\n            # ColumnTransformer). These are always lists/tuples of lists/\n            # tuples, size bigger than 2 and an OpenMLFlow item involved.\n            if not isinstance(values, (tuple, list)):\n                return False\n            for item in values:\n                if not isinstance(item, (tuple, list)):\n                    return False\n                if len(item) &lt; 2:\n                    return False\n                if not isinstance(item[1], openml.flows.OpenMLFlow):\n                    return False\n            return True\n\n        # _flow is openml flow object, _param dict maps from flow name to flow\n        # id for the main call, the param dict can be overridden (useful for\n        # unit tests / sentinels) this way, for flows without subflows we do\n        # not have to rely on _flow_dict\n        exp_parameters = set(_flow.parameters)\n        exp_components = set(_flow.components)\n        model_parameters = set([mp for mp in self._get_module_descriptors(component_model)\n                                if '__' not in mp])\n        if len((exp_parameters | exp_components) ^ model_parameters) != 0:\n            flow_params = sorted(exp_parameters | exp_components)\n            model_params = sorted(model_parameters)\n            raise ValueError('Parameters of the model do not match the '\n                             'parameters expected by the '\n                             'flow:\\nexpected flow parameters: '\n                             '%s\\nmodel parameters: %s' % (flow_params,\n                                                           model_params))\n\n        _params = []\n        for _param_name in _flow.parameters:\n            _current = OrderedDict()\n            _current['oml:name'] = _param_name\n\n            current_param_values = self.model_to_flow(\n                self._get_module_descriptors(component_model)[_param_name])\n\n            # Try to filter out components (a.k.a. subflows) which are\n            # handled further down in the code (by recursively calling\n            # this function)!\n            if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                continue\n\n            if is_subcomponent_specification(current_param_values):\n                # complex parameter value, with subcomponents\n                parsed_values = list()\n                for subcomponent in current_param_values:\n                    if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                        raise ValueError('Component reference should be '\n                                         'size {2,3}. ')\n\n                    subcomponent_identifier = subcomponent[0]\n                    subcomponent_flow = subcomponent[1]\n                    if not isinstance(subcomponent_identifier, str):\n                        raise TypeError('Subcomponent identifier should be '\n                                        'string')\n                    if not isinstance(subcomponent_flow,\n                                      openml.flows.OpenMLFlow):\n                        raise TypeError('Subcomponent flow should be string')\n\n                    current = {\n                        \"oml-python:serialized_object\": \"component_reference\",\n                        \"value\": {\n                            \"key\": subcomponent_identifier,\n                            \"step_name\": subcomponent_identifier\n                        }\n                    }\n                    if len(subcomponent) == 3:\n                        if not isinstance(subcomponent[2], list):\n                            raise TypeError('Subcomponent argument should be'\n                                            'list')\n                        current['value']['argument_1'] = subcomponent[2]\n                    parsed_values.append(current)\n                parsed_values = json.dumps(parsed_values)\n            else:\n                # vanilla parameter value\n                parsed_values = json.dumps(current_param_values)\n\n            _current['oml:value'] = parsed_values\n            if _main_call:\n                _current['oml:component'] = main_id\n            else:\n                _current['oml:component'] = _flow_dict[_flow.name]\n            _params.append(_current)\n\n        for _identifier in _flow.components:\n            subcomponent_model = self._get_module_descriptors(component_model)[_identifier]\n            _params.extend(extract_parameters(_flow.components[_identifier],\n                                              _flow_dict, subcomponent_model))\n        return _params\n\n    flow_dict = get_flow_dict(flow)\n    model = model if model is not None else flow.model\n    parameters = extract_parameters(flow, flow_dict, model, True, flow.flow_id)\n\n    return parameters\n</code></pre>"},{"location":"pytorch/API%20reference/OpenML%20Connection/#extension.PytorchExtension.seed_model","title":"<code>seed_model(model, seed=None)</code>","text":"<p>Set the random state of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is raised when a RandomState was used as seed).</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>pytorch model</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> <p>The seed to initialize the RandomState with. Unseeded subcomponents will be seeded with a random number from the RandomState.</p> <code>None</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>temp_dir/pytorch/openml_pytorch/extension.py</code> <pre><code>def seed_model(self, model: Any, seed: Optional[int] = None) -&gt; Any:\n    \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n    model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Models that are already seeded will maintain the seed. In this case,\n    only integer seeds are allowed (An exception is raised when a RandomState was used as\n    seed).\n\n    Parameters\n    ----------\n    model : pytorch model\n        The model to be seeded\n    seed : int\n        The seed to initialize the RandomState with. Unseeded subcomponents\n        will be seeded with a random number from the RandomState.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    return model\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/","title":"Trainer","text":"<p>This module provides classes and methods to facilitate the configuration, data handling, training, and evaluation of machine learning models using PyTorch and OpenML datasets. The functionalities include: - Generation of default configurations for models. - Handling of image and tabular data. - Training and evaluating machine learning models. - Exporting trained models to ONNX format. - Managing data transformations and loaders.</p> <p>This module provides classes and methods to facilitate the configuration, data handling, training, and evaluation of machine learning models using PyTorch and OpenML datasets. The functionalities include: - Generation of default configurations for models. - Handling of image and tabular data. - Training and evaluating machine learning models. - Exporting trained models to ONNX format. - Managing data transformations and loaders.</p>"},{"location":"pytorch/API%20reference/Trainer/#trainer.BaseDataHandler","title":"<code>BaseDataHandler</code>","text":"<p>BaseDataHandler class is an abstract base class for data handling operations.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class BaseDataHandler:\n    \"\"\"\n        BaseDataHandler class is an abstract base class for data handling operations.\n    \"\"\"\n    def prepare_data(self, X_train, y_train, X_val, y_val, data_config=None):\n        raise NotImplementedError\n\n    def prepare_test_data(self, X_test, data_config=None):\n        raise NotImplementedError\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DataContainer","title":"<code>DataContainer</code>","text":"<p>class DataContainer:     A class to contain the training, validation, and test data loaders. This just makes it easier to access them when required.</p> <pre><code>Attributes:\ntrain_dl: DataLoader object for the training data.\nvalid_dl: DataLoader object for the validation data.\ntest_dl: Optional DataLoader object for the test data.\n</code></pre> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class DataContainer:\n    \"\"\"\n    class DataContainer:\n        A class to contain the training, validation, and test data loaders. This just makes it easier to access them when required.\n\n        Attributes:\n        train_dl: DataLoader object for the training data.\n        valid_dl: DataLoader object for the validation data.\n        test_dl: Optional DataLoader object for the test data.\n    \"\"\"\n    def __init__(self, train_dl, valid_dl, test_dl=None):\n        self.train_dl, self.valid_dl = train_dl, valid_dl\n        self.test_dl = test_dl\n\n    @property\n    def train_ds(self):\n        return self.train_dl.dataset\n\n    @property\n    def valid_ds(self):\n        return self.valid_dl.dataset\n\n    @property\n    def test_ds(self):\n        return self.test_dl.dataset\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator","title":"<code>DefaultConfigGenerator</code>","text":"<p>DefaultConfigGenerator class provides various methods to generate default configurations.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class DefaultConfigGenerator:\n    \"\"\"\n    DefaultConfigGenerator class provides various methods to generate default configurations.\n    \"\"\"\n\n    @staticmethod\n    def _default_criterion_gen(task: OpenMLTask) -&gt; torch.nn.Module:\n        \"\"\"\n        _default_criterion_gen returns a criterion based on the task type - regressions use\n        torch.nn.SmoothL1Loss while classifications use torch.nn.CrossEntropyLoss\n        \"\"\"\n        if isinstance(task, OpenMLRegressionTask):\n            return torch.nn.SmoothL1Loss()\n        elif isinstance(task, OpenMLClassificationTask):\n            return torch.nn.CrossEntropyLoss()\n        else:\n            raise ValueError(task)\n\n    @staticmethod\n    def _default_optimizer_gen(model: torch.nn.Module, _: OpenMLTask):\n        \"\"\"\n        _default_optimizer_gen returns the torch.optim.Adam optimizer for the given model\n        \"\"\"\n        return torch.optim.Adam\n\n    @staticmethod\n    def _default_scheduler_gen(optim, _: OpenMLTask) -&gt; Any:\n        \"\"\"\n        _default_scheduler_gen returns the torch.optim.lr_scheduler.ReduceLROnPlateau scheduler for the given optimizer\n        \"\"\"\n        return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optim)\n\n    @staticmethod\n    def _default_predict(output: torch.Tensor, task: OpenMLTask) -&gt; torch.Tensor:\n        \"\"\"\n        _default_predict turns the outputs into predictions by returning the argmax of the output tensor for classification tasks, and by flattening the prediction in case of the regression\n        \"\"\"\n        output_axis = output.dim() - 1\n        if isinstance(task, OpenMLClassificationTask):\n            output = torch.argmax(output, dim=output_axis)\n        elif isinstance(task, OpenMLRegressionTask):\n            output = output.view(-1)\n        else:\n            raise ValueError(task)\n        return output\n\n    @staticmethod\n    def _default_predict_proba(\n        output: torch.Tensor, task: OpenMLTask\n    ) -&gt; torch.Tensor:\n        \"\"\"\n        _default_predict_proba turns the outputs into probabilities using softmax\n        \"\"\"\n        output_axis = output.dim() - 1\n        output = output.softmax(dim=output_axis)\n        return output\n\n    @staticmethod\n    def _default_sanitize(tensor: torch.Tensor) -&gt; torch.Tensor:\n        \"\"\"\n        _default sanitizer replaces NaNs with 1e-6\n        \"\"\"\n        tensor = torch.where(\n            torch.isnan(tensor), torch.ones_like(tensor) * torch.tensor(1e-6), tensor\n        )\n        return tensor\n\n    @staticmethod\n    def _default_retype_labels(\n         tensor: torch.Tensor, task: OpenMLTask\n    ) -&gt; torch.Tensor:\n        \"\"\"\n        _default_retype_labels changes the type of the tensor to long for classification tasks and to float for regression tasks\n        \"\"\"\n        if isinstance(task, OpenMLClassificationTask):\n            return tensor.long()\n        elif isinstance(task, OpenMLRegressionTask):\n            return tensor.float()\n        else:\n            raise ValueError(task)\n\n    def get_device(\n        self,\n    ):\n        \"\"\"\n        Checks if a GPU is available and returns the device to be used for training (cuda, mps or cpu)\n        \"\"\"\n        if torch.cuda.is_available():\n            device = torch.device(\"cuda\")\n        elif torch.backends.mps.is_available() and torch.backends.mps.is_built():\n            device = torch.device(\"mps\")\n        else:\n            device = torch.device(\"cpu\")\n\n        return device\n\n    def default_image_transform(self):\n        return Compose(\n            [\n                ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n                Lambda(convert_to_rgb),  # Convert PIL Image to RGB if it's not already.\n                Resize((128, 128)),  # Resize the image.\n                ToTensor(),  # Convert the PIL Image back to a tensor.\n            ]\n        )\n\n    def return_model_config(self):\n        \"\"\"\n        Returns a configuration object for the model\n        \"\"\"\n\n        return SimpleNamespace(\n            device=self.get_device(),\n            criterion=self._default_criterion_gen,\n            optimizer_gen=self._default_optimizer_gen,\n            scheduler_gen=self._default_scheduler_gen,\n            # predict turns the outputs of the model into actual predictions\n            predict=self._default_predict,  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n            # predict_proba turns the outputs of the model into probabilities for each class\n            predict_proba=self._default_predict_proba,  # type: Callable[[torch.Tensor], torch.Tensor]\n            # epoch_count represents the number of epochs the model should be trained for\n            epoch_count=3,  # type: int,\n            # progress_callback=(\n            #     self._default_progress_callback\n            # ),  # type: Callable[[int, int, int, int, float, float], None]\n            # enable progress bar\n            verbose=True,\n        )\n\n    def return_data_config(self):\n        \"\"\"\n        Returns a configuration object for the data\n        \"\"\"\n        return SimpleNamespace(\n            type_of_data=\"image\",\n            perform_validation=False,\n            # progress_callback is called when a training step is finished, in order to report the current progress\n            # sanitize sanitizes the input data in order to ensure that models can be trained safely\n            sanitize=self._default_sanitize,  # type: Callable[[torch.Tensor], torch.Tensor]\n            # retype_labels changes the types of the labels in order to ensure type compatibility\n            retype_labels=(\n                self._default_retype_labels\n            ),  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n            # image_size is the size of the images that are fed into the model\n            image_size=128,\n            # batch_size represents the processing batch size for training\n            batch_size=64,  # type: int\n            data_augmentation=None,\n            validation_split=0.1,\n            transform=self.default_image_transform(),\n        )\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator.get_device","title":"<code>get_device()</code>","text":"<p>Checks if a GPU is available and returns the device to be used for training (cuda, mps or cpu)</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def get_device(\n    self,\n):\n    \"\"\"\n    Checks if a GPU is available and returns the device to be used for training (cuda, mps or cpu)\n    \"\"\"\n    if torch.cuda.is_available():\n        device = torch.device(\"cuda\")\n    elif torch.backends.mps.is_available() and torch.backends.mps.is_built():\n        device = torch.device(\"mps\")\n    else:\n        device = torch.device(\"cpu\")\n\n    return device\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator.return_data_config","title":"<code>return_data_config()</code>","text":"<p>Returns a configuration object for the data</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def return_data_config(self):\n    \"\"\"\n    Returns a configuration object for the data\n    \"\"\"\n    return SimpleNamespace(\n        type_of_data=\"image\",\n        perform_validation=False,\n        # progress_callback is called when a training step is finished, in order to report the current progress\n        # sanitize sanitizes the input data in order to ensure that models can be trained safely\n        sanitize=self._default_sanitize,  # type: Callable[[torch.Tensor], torch.Tensor]\n        # retype_labels changes the types of the labels in order to ensure type compatibility\n        retype_labels=(\n            self._default_retype_labels\n        ),  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n        # image_size is the size of the images that are fed into the model\n        image_size=128,\n        # batch_size represents the processing batch size for training\n        batch_size=64,  # type: int\n        data_augmentation=None,\n        validation_split=0.1,\n        transform=self.default_image_transform(),\n    )\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.DefaultConfigGenerator.return_model_config","title":"<code>return_model_config()</code>","text":"<p>Returns a configuration object for the model</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def return_model_config(self):\n    \"\"\"\n    Returns a configuration object for the model\n    \"\"\"\n\n    return SimpleNamespace(\n        device=self.get_device(),\n        criterion=self._default_criterion_gen,\n        optimizer_gen=self._default_optimizer_gen,\n        scheduler_gen=self._default_scheduler_gen,\n        # predict turns the outputs of the model into actual predictions\n        predict=self._default_predict,  # type: Callable[[torch.Tensor, OpenMLTask], torch.Tensor]\n        # predict_proba turns the outputs of the model into probabilities for each class\n        predict_proba=self._default_predict_proba,  # type: Callable[[torch.Tensor], torch.Tensor]\n        # epoch_count represents the number of epochs the model should be trained for\n        epoch_count=3,  # type: int,\n        # progress_callback=(\n        #     self._default_progress_callback\n        # ),  # type: Callable[[int, int, int, int, float, float], None]\n        # enable progress bar\n        verbose=True,\n    )\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.OpenMLImageHandler","title":"<code>OpenMLImageHandler</code>","text":"<p>               Bases: <code>BaseDataHandler</code></p> <p>OpenMLImageHandler is a class that extends BaseDataHandler to handle image data from OpenML datasets.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class OpenMLImageHandler(BaseDataHandler):\n    \"\"\"\n        OpenMLImageHandler is a class that extends BaseDataHandler to handle image data from OpenML datasets.\n    \"\"\"\n    def prepare_data(self, X_train, y_train, X_val, y_val, data_config = None):\n        train = OpenMLImageDataset(\n            image_dir=data_config.file_dir,\n            X=X_train,\n            y=y_train,\n            transform_x=data_config.transform,\n            image_size=data_config.image_size,\n        )\n        val = OpenMLImageDataset(\n            image_dir=data_config.file_dir,\n            X=X_val,\n            y=y_val,\n            transform_x=data_config.transform,\n            image_size=data_config.image_size,\n        )\n        return train, val\n\n    def prepare_test_data(self, X_test, data_config = None):\n        test = OpenMLImageDataset(\n            image_dir=data_config.file_dir,\n            X=X_test,\n            y=None,\n            transform_x=data_config.transform,\n            image_size=data_config.image_size,\n        )\n        return test\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.OpenMLTabularHandler","title":"<code>OpenMLTabularHandler</code>","text":"<p>               Bases: <code>BaseDataHandler</code></p> <p>OpenMLTabularHandler is a class that extends BaseDataHandler to handle tabular data from OpenML datasets.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class OpenMLTabularHandler(BaseDataHandler):\n    \"\"\"\n    OpenMLTabularHandler is a class that extends BaseDataHandler to handle tabular data from OpenML datasets.\n    \"\"\"\n    def prepare_data(self, X_train, y_train, X_val, y_val, data_config = None):\n        train = OpenMLTabularDataset(X=X_train, y=y_train)\n        val = OpenMLTabularDataset(X=X_val, y=y_val)\n        return train, val\n\n    def prepare_test_data(self, X_test, data_config = None):\n        test = OpenMLTabularDataset(X=X_test, y=None)\n        return test\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.OpenMLTrainerModule","title":"<code>OpenMLTrainerModule</code>","text":"Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>class OpenMLTrainerModule:\n\n    def _default_progress_callback(\n        self, fold: int, rep: int, epoch: int, step: int, loss: float, accuracy: float\n    ):\n        # todo : move this into callback\n        \"\"\"\n                _default_progress_callback reports the current fold, rep, epoch, step and loss for every\n        training iteration to the default logger\n        \"\"\"\n        self.logger.info(\n            \"[%d, %d, %d, %d] loss: %.4f, accuracy: %.4f\"\n            % (fold, rep, epoch, step, loss, accuracy)\n        )\n    def __init__(\n        self,\n        data_module: OpenMLDataModule,\n        callbacks: List[Callback] = [],\n        **kwargs,\n    ):\n        self.config_gen = DefaultConfigGenerator()\n        self.model_config = self.config_gen.return_model_config()\n        self.data_module = data_module\n        self.callbacks = callbacks\n\n        self.config = SimpleNamespace(\n            **{**self.model_config.__dict__, **self.data_module.data_config.__dict__}\n        )\n        # update the config with the user defined values\n        self.config.__dict__.update(kwargs)\n        self.config.progress_callback = self._default_progress_callback\n        self.logger: logging.Logger = logging.getLogger(__name__)\n\n        self.user_defined_measures = OrderedDict()\n        # self.callbacks.append(LoggingCallback(self.logger, print_output=False))\n        self.loss = 0\n        self.training_state = True\n\n        self.phases = [0.2, 0.8]\n        self.scheds = combine_scheds(\n            self.phases, [sched_cos(1e-4, 5e-3), sched_cos(5e-3, 1e-3)]\n        )\n\n        self.cbfs = [\n            Recorder,\n            partial(AvgStatsCallBack, [accuracy]),\n            partial(ParamScheduler, \"lr\", self.scheds),\n            # TensorBoardCallback(),\n        ]\n\n    def _onnx_export(self, model_copy):\n        f = io.BytesIO()\n        torch.onnx.export(model_copy, sample_input, f)\n        onnx_model = onnx.load_model_from_string(f.getvalue())\n        onnx_ = onnx_model.SerializeToString()\n        return onnx_\n\n    def run_model_on_fold(\n        self,\n        model: torch.nn.Module,\n        task: OpenMLTask,\n        X_train: pd.DataFrame,\n        rep_no: int,\n        fold_no: int,\n        y_train: Optional[pd.Series],\n        X_test: pd.DataFrame,\n    ) -&gt; Tuple[np.ndarray, Optional[np.ndarray], OrderedDict, Optional[Any]]:\n\n        # if task has no class labels, we assign the class labels to be the unique values in the training set\n        if task.class_labels is None:\n            task.class_labels = y_train.unique()\n\n        self.add_callbacks()\n\n        self.model = copy.deepcopy(model)\n\n        try:\n            data, model_classes = self.run_training(task, X_train, y_train, X_test)\n\n        except AttributeError as e:\n            # typically happens when training a regressor8 on classification task\n            raise PyOpenMLError(str(e))\n\n        # In supervised learning this returns the predictions for Y\n        pred_y, proba_y = self.run_evaluation(task, data, model_classes)\n\n        # Convert predictions to class labels\n        if task.class_labels is not None:\n            pred_y = [task.class_labels[i] for i in pred_y]\n\n        # Convert model to onnx\n        onnx_ = self._onnx_export(self.model)\n\n        global last_models\n        last_models = onnx_\n\n        return pred_y, proba_y, self.user_defined_measures, None\n\n    def check_config(self):\n        raise NotImplementedError\n\n    def _prediction_to_probabilities(\n        self, y: np.ndarray, classes: List[Any]\n    ) -&gt; np.ndarray:\n        \"\"\"Transforms predicted probabilities to match with OpenML class indices.\n\n        Parameters\n        ----------\n        y : np.ndarray\n            Predicted probabilities (possibly omitting classes if they were not present in the\n            training data).\n        model_classes : list\n            List of classes known_predicted by the model, ordered by their index.\n\n        Returns\n        -------\n        np.ndarray\n        \"\"\"\n        # y: list or numpy array of predictions\n        # model_classes: mapping from original array id to\n        # prediction index id\n        if not isinstance(classes, list):\n            raise ValueError(\n                \"please convert model classes to list prior to \" \"calling this fn\"\n            )\n        result = np.zeros((len(y), len(classes)), dtype=np.float32)\n        for obs, prediction_idx in enumerate(y):\n            result[obs][prediction_idx] = 1.0\n        return result\n\n    def run_evaluation(self, task, data, model_classes):\n        if isinstance(task, OpenMLSupervisedTask):\n            self.model.eval()\n            pred_y = self.pred_test(task, self.model, data.test_dl, self.config.predict)\n        else:\n            raise ValueError(task)\n\n        if isinstance(task, OpenMLClassificationTask):\n            try:\n                self.model.eval()\n                proba_y = self.pred_test(\n                    task, self.model, data.test_dl, self.config.predict_proba\n                )\n\n            except AttributeError:\n                if task.class_labels is not None:\n                    proba_y = self._prediction_to_probabilities(\n                        pred_y, list(task.class_labels)\n                    )\n                else:\n                    raise ValueError(\"The task has no class labels\")\n\n            if task.class_labels is not None:\n                if proba_y.shape[1] != len(task.class_labels):\n                    # Remap the probabilities in case there was a class missing\n                    # at training time. By default, the classification targets\n                    # are mapped to be zero-based indices to the actual classes.\n                    # Therefore, the model_classes contain the correct indices to\n                    # the correct probability array. Example:\n                    # classes in the dataset: 0, 1, 2, 3, 4, 5\n                    # classes in the training set: 0, 1, 2, 4, 5\n                    # then we need to add a column full of zeros into the probabilities\n                    # for class 3 because the rest of the library expects that the\n                    # probabilities are ordered the same way as the classes are ordered).\n                    proba_y_new = np.zeros((proba_y.shape[0], len(task.class_labels)))\n                    for idx, model_class in enumerate(model_classes):\n                        proba_y_new[:, model_class] = proba_y[:, idx]\n                    proba_y = proba_y_new\n\n                if proba_y.shape[1] != len(task.class_labels):\n                    message = \"Estimator only predicted for {}/{} classes!\".format(\n                        proba_y.shape[1],\n                        len(task.class_labels),\n                    )\n                    warnings.warn(message)\n                    self.logger.warning(message)\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            proba_y = None\n\n        else:\n            raise TypeError(type(task))\n        return pred_y, proba_y\n\n    def run_training(self, task, X_train, y_train, X_test):\n        if isinstance(task, OpenMLSupervisedTask) or isinstance(\n            task, OpenMLClassificationTask\n        ):\n            self.opt = self.config.optimizer_gen(self.model, task)(\n                self.model.parameters()\n            )\n\n            self.criterion = self.config.criterion(task)\n            self.device = self.config.device\n\n            if self.config.device != \"cpu\":\n                self.criterion = self.criterion.to(self.config.device)\n\n            data, model_classes = self.data_module.get_data(\n                X_train, y_train, X_test, task\n            )\n            self.learn = Learner(\n                self.model,\n                self.opt,\n                self.criterion,\n                data,\n                model_classes,\n            )\n            self.learn.device = self.device\n            self.learn.model.to(self.device)\n            gc.collect()\n\n            self.runner = ModelRunner(cb_funcs=self.cbfs)\n            self.learn.model.train()\n            self.runner.fit(epochs=self.config.epoch_count, learn=self.learn)\n            self.learn.model.eval()\n\n            print(\"Loss\", self.runner.loss)\n        return data, model_classes\n\n    def add_callbacks(self):\n        if self.callbacks is not None and len(self.callbacks) &gt; 0:\n            for callback in self.callbacks:\n                if callback not in self.cbfs:\n                    self.cbfs.append(callback)\n\n    def pred_test(self, task, model_copy, test_loader, predict_func):\n        probabilities = []\n        for batch_idx, inputs in enumerate(test_loader):\n            inputs = self.config.sanitize(inputs)\n            # if torch.cuda.is_available():\n            inputs = inputs.to(self.config.device)\n\n            # Perform inference on the batch\n            pred_y_batch = model_copy(inputs)\n            pred_y_batch = predict_func(pred_y_batch, task)\n            pred_y_batch = pred_y_batch.cpu().detach().numpy()\n\n            probabilities.append(pred_y_batch)\n\n            # Concatenate probabilities from all batches\n        pred_y = np.concatenate(probabilities, axis=0)\n        return pred_y\n</code></pre>"},{"location":"pytorch/API%20reference/Trainer/#trainer.convert_to_rgb","title":"<code>convert_to_rgb(image)</code>","text":"<p>Converts an image to RGB mode if it is not already in that mode.</p> <p>Parameters: image (PIL.Image): The image to be converted.</p> <p>Returns: PIL.Image: The converted image in RGB mode.</p> Source code in <code>temp_dir/pytorch/openml_pytorch/trainer.py</code> <pre><code>def convert_to_rgb(image):\n    \"\"\"\n        Converts an image to RGB mode if it is not already in that mode.\n\n        Parameters:\n        image (PIL.Image): The image to be converted.\n\n        Returns:\n        PIL.Image: The converted image in RGB mode.\n    \"\"\"\n    if image.mode != \"RGB\":\n        return image.convert(\"RGB\")\n    return image\n</code></pre>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/","title":"Create dataset and task - tiniest imagenet","text":"In\u00a0[2]: Copied! <pre>import openml\n\nimport numpy as np\nimport pandas as pd\nimport sklearn.datasets\n\nimport openml\nfrom openml.datasets.functions import create_dataset\nimport os\nimport requests\nimport zipfile\nimport glob\n</pre> import openml  import numpy as np import pandas as pd import sklearn.datasets  import openml from openml.datasets.functions import create_dataset import os import requests import zipfile import glob In\u00a0[9]: Copied! <pre>def create_tiny_imagenet():\n    dir_name = \"datasets\"\n    os.makedirs(dir_name, exist_ok=True)\n\n    # download the dataset\n    url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"\n    r = requests.get(url, stream=True)\n\n    if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):\n        with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:\n            f.write(r.content)\n\n        with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:\n            zip_ref.extractall(f\"{dir_name}/\")\n    ## recusively find all the images\n    image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")\n    ## remove the first part of the path\n    image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]\n    ## create a dataframe with the image path and the label\n    label_func = lambda x: x.split(\"/\")[2]\n    df = pd.DataFrame(image_paths, columns=[\"image_path\"])\n    df[\"label\"] = df[\"image_path\"].apply(label_func)\n    ## encode the labels as integers\n    # df[\"Class_encoded\"] = pd.factorize(df[\"label\"])[0]\n\n    ## encode types\n    df[\"image_path\"] = df[\"image_path\"].astype(\"string\")\n    df[\"label\"] = df[\"label\"].astype(\"string\")\n\n\n    name = \"tiny-imagenet-200\"\n    attribute_names = df.columns\n    description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"\n    paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"\n    citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")\n\n    tinyim = create_dataset(\n        name = name,\n        description = description,\n        creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        collection_date = \"2017\",\n        language= \"English\",\n        licence=\"DbCL v1.0\",\n        default_target_attribute=\"label\",\n        attributes=\"auto\",\n        data=df,\n        citation=citation,\n        ignore_attribute=None\n    )\n    openml.config.apikey = ''\n    tinyim.publish()\n    print(f\"URL for dataset: {tinyim.openml_url}\")\n</pre> def create_tiny_imagenet():     dir_name = \"datasets\"     os.makedirs(dir_name, exist_ok=True)      # download the dataset     url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"     r = requests.get(url, stream=True)      if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):         with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:             f.write(r.content)          with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:             zip_ref.extractall(f\"{dir_name}/\")     ## recusively find all the images     image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")     ## remove the first part of the path     image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]     ## create a dataframe with the image path and the label     label_func = lambda x: x.split(\"/\")[2]     df = pd.DataFrame(image_paths, columns=[\"image_path\"])     df[\"label\"] = df[\"image_path\"].apply(label_func)     ## encode the labels as integers     # df[\"Class_encoded\"] = pd.factorize(df[\"label\"])[0]      ## encode types     df[\"image_path\"] = df[\"image_path\"].astype(\"string\")     df[\"label\"] = df[\"label\"].astype(\"string\")       name = \"tiny-imagenet-200\"     attribute_names = df.columns     description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"     paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"     citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")      tinyim = create_dataset(         name = name,         description = description,         creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         collection_date = \"2017\",         language= \"English\",         licence=\"DbCL v1.0\",         default_target_attribute=\"label\",         attributes=\"auto\",         data=df,         citation=citation,         ignore_attribute=None     )     openml.config.apikey = ''     tinyim.publish()     print(f\"URL for dataset: {tinyim.openml_url}\")  In\u00a0[10]: Copied! <pre>create_tiny_imagenet()\n# https://www.openml.org/d/46346\n</pre> create_tiny_imagenet() # https://www.openml.org/d/46346 In\u00a0[19]: Copied! <pre>def create_tiniest_imagenet():\n    dir_name = \"datasets\"\n    os.makedirs(dir_name, exist_ok=True)\n\n    # download the dataset\n    url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"\n    r = requests.get(url, stream=True)\n\n    if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):\n        with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:\n            f.write(r.content)\n\n        with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:\n            zip_ref.extractall(f\"{dir_name}/\")\n    ## recusively find all the images\n    image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")\n    ## remove the first part of the path\n    image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]\n    image_paths[-1]\n    ## create a dataframe with the image path and the label\n    label_func = lambda x: x.split(\"/\")[2]\n    df = pd.DataFrame(image_paths, columns=[\"image_path\"])\n    df[\"label\"] = df[\"image_path\"].apply(label_func)\n    ## encode types\n    df[\"image_path\"] = df[\"image_path\"].astype(\"string\")\n    df[\"label\"] = df[\"label\"].astype(\"string\")\n\n    # keep only first 20 images for each label\n    df = df.groupby(\"label\").head(20)\n\n\n    name = \"tiniest-imagenet-200\"\n    attribute_names = df.columns\n    description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. !!! This dataset only links to 20 images per class (instead of the usual 500) and is ONLY for quickly testing a framework. !!! Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"\n    paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"\n    citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")\n\n    tinyim = create_dataset(\n        name = name,\n        description = description,\n        creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",\n        collection_date = \"2017\",\n        language= \"English\",\n        licence=\"DbCL v1.0\",\n        default_target_attribute=\"label\",\n        attributes=\"auto\",\n        data=df,\n        citation=citation,\n        ignore_attribute=None\n    )\n    openml.config.apikey = ''\n    tinyim.publish()\n    print(f\"URL for dataset: {tinyim.openml_url}\")\n</pre> def create_tiniest_imagenet():     dir_name = \"datasets\"     os.makedirs(dir_name, exist_ok=True)      # download the dataset     url = \"http://cs231n.stanford.edu/tiny-imagenet-200.zip\"     r = requests.get(url, stream=True)      if not os.path.exists(f\"{dir_name}/tiny-imagenet-200.zip\"):         with open(f\"{dir_name}/tiny-imagenet-200.zip\", \"wb\") as f:             f.write(r.content)          with zipfile.ZipFile(f\"{dir_name}/tiny-imagenet-200.zip\", 'r') as zip_ref:             zip_ref.extractall(f\"{dir_name}/\")     ## recusively find all the images     image_paths = glob.glob(f\"{dir_name}/tiny-imagenet-200/train/*/*/*.JPEG\")     ## remove the first part of the path     image_paths = [path.split(\"/\", 1)[-1] for path in image_paths]     image_paths[-1]     ## create a dataframe with the image path and the label     label_func = lambda x: x.split(\"/\")[2]     df = pd.DataFrame(image_paths, columns=[\"image_path\"])     df[\"label\"] = df[\"image_path\"].apply(label_func)     ## encode types     df[\"image_path\"] = df[\"image_path\"].astype(\"string\")     df[\"label\"] = df[\"label\"].astype(\"string\")      # keep only first 20 images for each label     df = df.groupby(\"label\").head(20)       name = \"tiniest-imagenet-200\"     attribute_names = df.columns     description = \"Tiny ImageNet contains 100000 images of 200 classes (500 for each class) downsized to 64 x 64 colored images. !!! This dataset only links to 20 images per class (instead of the usual 500) and is ONLY for quickly testing a framework. !!! Each class has 500 training images, 50 validation images, and 50 test images. The dataset here just contains links to the images and the labels. The dataset can be downloaded from the official website ![here](http://cs231n.stanford.edu/tiny-imagenet-200.zip). /n Link to the paper - [Tiny ImageNet Classification with CNN](https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf)\"     paper_url = \"https://cs231n.stanford.edu/reports/2017/pdfs/930.pdf\"     citation = (\"Wu, J., Zhang, Q., &amp; Xu, G. (2017). Tiny imagenet challenge. Technical report.\")      tinyim = create_dataset(         name = name,         description = description,         creator= \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         contributor = \"Jiayu Wu, Qixiang Zhang, Guoxi Xu\",         collection_date = \"2017\",         language= \"English\",         licence=\"DbCL v1.0\",         default_target_attribute=\"label\",         attributes=\"auto\",         data=df,         citation=citation,         ignore_attribute=None     )     openml.config.apikey = ''     tinyim.publish()     print(f\"URL for dataset: {tinyim.openml_url}\")  In\u00a0[20]: Copied! <pre>create_tiniest_imagenet()\n# https://www.openml.org/d/46347\n</pre> create_tiniest_imagenet() # https://www.openml.org/d/46347 <pre>URL for dataset: https://www.openml.org/d/46347\n</pre> In\u00a0[27]: Copied! <pre>def create_task():\n    # Define task parameters\n    task_type = openml.tasks.TaskType.SUPERVISED_CLASSIFICATION\n    dataset_id = 46347 # Obtained from the dataset creation step\n    evaluation_measure = 'predictive_accuracy'\n    target_name = 'label'\n    class_labels = list(pd.read_csv(\"datasets/tiniest_imagenet.csv\")[\"label\"].unique())\n    cost_matrix = None\n\n    # Create the task\n    new_task = openml.tasks.create_task(\n        task_type=task_type,\n        dataset_id=dataset_id, \n        estimation_procedure_id = 1,\n        evaluation_measure=evaluation_measure,\n        target_name=target_name,\n        class_labels=class_labels,\n        cost_matrix=cost_matrix\n    )\n    openml.config.apikey = ''\n    new_task.publish()\n    print(f\"URL for task: {new_task.openml_url}\")\n</pre> def create_task():     # Define task parameters     task_type = openml.tasks.TaskType.SUPERVISED_CLASSIFICATION     dataset_id = 46347 # Obtained from the dataset creation step     evaluation_measure = 'predictive_accuracy'     target_name = 'label'     class_labels = list(pd.read_csv(\"datasets/tiniest_imagenet.csv\")[\"label\"].unique())     cost_matrix = None      # Create the task     new_task = openml.tasks.create_task(         task_type=task_type,         dataset_id=dataset_id,          estimation_procedure_id = 1,         evaluation_measure=evaluation_measure,         target_name=target_name,         class_labels=class_labels,         cost_matrix=cost_matrix     )     openml.config.apikey = ''     new_task.publish()     print(f\"URL for task: {new_task.openml_url}\") In\u00a0[28]: Copied! <pre>create_task()\n# https://www.openml.org/t/362128\n</pre> create_task() # https://www.openml.org/t/362128 <pre>URL for task: https://www.openml.org/t/362128\n</pre>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#create-dataset-and-task-tiniest-imagenet","title":"Create dataset and task - tiniest imagenet\u00b6","text":"<ul> <li>An example of how to create a custom dataset and task using the OpenML API and upload it to the OpenML server.</li> <li>Note that you must have an API key from the OpenML website to upload datasets and tasks.</li> </ul>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#create-dataset-on-openml","title":"Create dataset on OpenML\u00b6","text":"<ul> <li>Instead of making our own, we obtain a subset of the ImageNet dataset from Stanford. This dataset has 200 classes.</li> </ul>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#another-even-tinier-dataset","title":"Another, even tinier dataset\u00b6","text":"<ul> <li>We subset the previous dataset to 20 images per class.</li> </ul>"},{"location":"pytorch/Examples/Create%20Dataset%20and%20Task/#create-task-on-openml","title":"Create task on OpenML\u00b6","text":"<ul> <li>Now to actually use the OpenML Pytorch API, we need to have a task associated with the dataset. This is how we create it.</li> </ul>"},{"location":"pytorch/Examples/Image%20Classification%20Task/","title":"Image classification task","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml_pytorch.config\nimport openml\nimport logging\nimport warnings\n\n# Suppress FutureWarning messages\nwarnings.simplefilter(action='ignore')\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n\n############################################################################\nfrom openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda\nimport torchvision\n\nfrom openml_pytorch.trainer import convert_to_rgb\n</pre> import torch.nn import torch.optim  import openml_pytorch.config import openml import logging import warnings  # Suppress FutureWarning messages warnings.simplefilter(action='ignore')  ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################  ############################################################################ from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule from torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda import torchvision  from openml_pytorch.trainer import convert_to_rgb In\u00a0[\u00a0]: Copied! <pre>model = torchvision.models.efficientnet_b0(num_classes=200)\n</pre> model = torchvision.models.efficientnet_b0(num_classes=200) In\u00a0[\u00a0]: Copied! <pre>transform = Compose(\n    [\n        ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n        Lambda(\n            convert_to_rgb\n        ),  # Convert PIL Image to RGB if it's not already.\n        Resize(\n            (64, 64)\n        ),  # Resize the image.\n        ToTensor(),  # Convert the PIL Image back to a tensor.\n    ]\n)\ndata_module = OpenMLDataModule(\n    type_of_data=\"image\",\n    file_dir=\"datasets\",\n    filename_col=\"image_path\",\n    target_mode=\"categorical\",\n    target_column=\"label\",\n    batch_size = 64,\n    transform=transform\n)\n</pre> transform = Compose(     [         ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.         Lambda(             convert_to_rgb         ),  # Convert PIL Image to RGB if it's not already.         Resize(             (64, 64)         ),  # Resize the image.         ToTensor(),  # Convert the PIL Image back to a tensor.     ] ) data_module = OpenMLDataModule(     type_of_data=\"image\",     file_dir=\"datasets\",     filename_col=\"image_path\",     target_mode=\"categorical\",     target_column=\"label\",     batch_size = 64,     transform=transform ) In\u00a0[\u00a0]: Copied! <pre>trainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n    callbacks=[],\n)\nopenml_pytorch.config.trainer = trainer\n</pre> trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 1,     callbacks=[], ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre># Download the OpenML task for tiniest imagenet\ntask = openml.tasks.get_task(362128)\n</pre> # Download the OpenML task for tiniest imagenet task = openml.tasks.get_task(362128) In\u00a0[\u00a0]: Copied! <pre>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr() In\u00a0[\u00a0]: Copied! <pre>trainer.learn.model_classes\n</pre> trainer.learn.model_classes In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish()"},{"location":"pytorch/Examples/Image%20Classification%20Task/#image-classification-task","title":"Image classification task\u00b6","text":"<ul> <li>Image classification on OpenML Task (362128), tiniest ImageNet dataset.</li> </ul>"},{"location":"pytorch/Examples/Image%20Classification%20Task/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the data is present in the <code>file_dir</code> directory, and the <code>filename_col</code> is correctly set along with this column correctly pointing to where your data is stored.</li> </ul>"},{"location":"pytorch/Examples/Image%20Classification%20Task/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#download-the-task","title":"Download the task\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#view-the-classes-in-the-model","title":"View the classes in the model\u00b6","text":""},{"location":"pytorch/Examples/Image%20Classification%20Task/#publish-the-run-to-openml","title":"Publish the run to OpenML\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/","title":"Pretrained Image classification example - Transformer","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml\nimport openml_pytorch\nimport openml_pytorch.layers\nimport openml_pytorch.config\nfrom openml import OpenMLTask\nimport logging\nimport warnings\nfrom torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda\nfrom openml_pytorch.trainer import convert_to_rgb\n# Suppress FutureWarning messages\nwarnings.simplefilter(action='ignore')\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n\n############################################################################\nimport torch.nn as nn\nimport torch.nn.functional as F\n</pre> import torch.nn import torch.optim  import openml import openml_pytorch import openml_pytorch.layers import openml_pytorch.config from openml import OpenMLTask import logging import warnings from torchvision.transforms import Compose, Resize, ToPILImage, ToTensor, Lambda from openml_pytorch.trainer import convert_to_rgb # Suppress FutureWarning messages warnings.simplefilter(action='ignore')  ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################  ############################################################################ import torch.nn as nn import torch.nn.functional as F In\u00a0[\u00a0]: Copied! <pre># openml.config.apikey = 'key'\nfrom openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom openml_pytorch.trainer import Callback\n</pre> # openml.config.apikey = 'key' from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule from openml_pytorch.trainer import Callback In\u00a0[\u00a0]: Copied! <pre># Example model. You can do better :)\nimport torchvision.models as models\n\n# Load the pre-trained ResNet model\nmodel = models.efficientnet_b0(pretrained=True)\n\n# Modify the last fully connected layer to the required number of classes\nnum_classes = 200\nin_features = model.classifier[-1].in_features\n# model.fc = nn.Linear(in_features, num_classes)\nmodel.classifier = nn.Sequential(\n    nn.Dropout(p=0.2, inplace=True),\n    nn.Linear(in_features, num_classes),\n)\n\n# Optional: If you're fine-tuning, you may want to freeze the pre-trained layers\n# for param in model.parameters():\n#     param.requires_grad = False\n\n# # If you want to train the last layer only (the newly added layer)\n# for param in model.fc.parameters():\n#     param.requires_grad = True\n</pre> # Example model. You can do better :) import torchvision.models as models  # Load the pre-trained ResNet model model = models.efficientnet_b0(pretrained=True)  # Modify the last fully connected layer to the required number of classes num_classes = 200 in_features = model.classifier[-1].in_features # model.fc = nn.Linear(in_features, num_classes) model.classifier = nn.Sequential(     nn.Dropout(p=0.2, inplace=True),     nn.Linear(in_features, num_classes), )  # Optional: If you're fine-tuning, you may want to freeze the pre-trained layers # for param in model.parameters(): #     param.requires_grad = False  # # If you want to train the last layer only (the newly added layer) # for param in model.fc.parameters(): #     param.requires_grad = True In\u00a0[\u00a0]: Copied! <pre>transform = Compose(\n    [\n        ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.\n        Lambda(\n            convert_to_rgb\n        ),  # Convert PIL Image to RGB if it's not already.\n        Resize(\n            (64, 64)\n        ),  # Resize the image.\n        ToTensor(),  # Convert the PIL Image back to a tensor.\n    ]\n)\ndata_module = OpenMLDataModule(\n    type_of_data=\"image\",\n    file_dir=\"datasets\",\n    filename_col=\"image_path\",\n    target_mode=\"categorical\",\n    target_column=\"label\",\n    batch_size = 64,\n    transform=transform\n)\n</pre> transform = Compose(     [         ToPILImage(),  # Convert tensor to PIL Image to ensure PIL Image operations can be applied.         Lambda(             convert_to_rgb         ),  # Convert PIL Image to RGB if it's not already.         Resize(             (64, 64)         ),  # Resize the image.         ToTensor(),  # Convert the PIL Image back to a tensor.     ] ) data_module = OpenMLDataModule(     type_of_data=\"image\",     file_dir=\"datasets\",     filename_col=\"image_path\",     target_mode=\"categorical\",     target_column=\"label\",     batch_size = 64,     transform=transform ) In\u00a0[\u00a0]: Copied! <pre>def custom_optimizer_gen(model: torch.nn.Module, task: OpenMLTask) -&gt; torch.optim.Optimizer:\n    return torch.optim.Adam(model.fc.parameters())\n\ntrainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n    optimizer = custom_optimizer_gen,\n    callbacks=[],\n)\nopenml_pytorch.config.trainer = trainer\n</pre> def custom_optimizer_gen(model: torch.nn.Module, task: OpenMLTask) -&gt; torch.optim.Optimizer:     return torch.optim.Adam(model.fc.parameters())  trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 1,     optimizer = custom_optimizer_gen,     callbacks=[], ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre># Download the OpenML task for tiniest imagenet\ntask = openml.tasks.get_task(362128)\n</pre>  # Download the OpenML task for tiniest imagenet task = openml.tasks.get_task(362128) In\u00a0[\u00a0]: Copied! <pre>#\n# Run the model on the task (requires an API key).m\nrun = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> # # Run the model on the task (requires an API key).m run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr() In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish()"},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#pretrained-image-classification-example-transformer","title":"Pretrained Image classification example - Transformer\u00b6","text":"<ul> <li>Pretrained image classification using a Transformer architecture, \"custom\" Optimizer for OpenML Task (362128) , tiniest ImageNet dataset.</li> </ul>"},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the data is present in the <code>file_dir</code> directory, and the <code>filename_col</code> is correctly set along with this column correctly pointing to where your data is stored.</li> </ul>"},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#download-the-task","title":"Download the task\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/#publish-the-run-to-openml","title":"Publish the run to OpenML\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/","title":"Sequential classification","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml_pytorch.config\nimport openml\nimport logging\nimport warnings\n\n# Suppress FutureWarning messages\nwarnings.simplefilter(action='ignore')\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n</pre>  import torch.nn import torch.optim  import openml_pytorch.config import openml import logging import warnings  # Suppress FutureWarning messages warnings.simplefilter(action='ignore')  ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################ In\u00a0[\u00a0]: Copied! <pre>from openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\n</pre> from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule In\u00a0[\u00a0]: Copied! <pre>############################################################################\n# Define a sequential network that does the initial image reshaping\n# and normalization model.\nprocessing_net = torch.nn.Sequential(\n    openml_pytorch.layers.Functional(function=torch.Tensor.reshape,\n                                                shape=(-1, 1, 28, 28)),\n    torch.nn.BatchNorm2d(num_features=1)\n)\n############################################################################\n\n############################################################################\n# Define a sequential network that does the extracts the features from the\n# image.\nfeatures_net = torch.nn.Sequential(\n    torch.nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5),\n    torch.nn.LeakyReLU(),\n    torch.nn.MaxPool2d(kernel_size=2),\n    torch.nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5),\n    torch.nn.LeakyReLU(),\n    torch.nn.MaxPool2d(kernel_size=2),\n)\n############################################################################\n\n############################################################################\n# Define a sequential network that flattens the features and compiles the\n# results into probabilities for each digit.\nresults_net = torch.nn.Sequential(\n    openml_pytorch.layers.Functional(function=torch.Tensor.reshape,\n                                                shape=(-1, 4 * 4 * 64)),\n    torch.nn.Linear(in_features=4 * 4 * 64, out_features=256),\n    torch.nn.LeakyReLU(),\n    torch.nn.Dropout(),\n    torch.nn.Linear(in_features=256, out_features=10),\n)\n############################################################################\n# openml.config.apikey = 'key'\n\n############################################################################\n# The main network, composed of the above specified networks.\nmodel = torch.nn.Sequential(\n    processing_net,\n    features_net,\n    results_net\n)\n############################################################################\n</pre>  ############################################################################ # Define a sequential network that does the initial image reshaping # and normalization model. processing_net = torch.nn.Sequential(     openml_pytorch.layers.Functional(function=torch.Tensor.reshape,                                                 shape=(-1, 1, 28, 28)),     torch.nn.BatchNorm2d(num_features=1) ) ############################################################################  ############################################################################ # Define a sequential network that does the extracts the features from the # image. features_net = torch.nn.Sequential(     torch.nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5),     torch.nn.LeakyReLU(),     torch.nn.MaxPool2d(kernel_size=2),     torch.nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5),     torch.nn.LeakyReLU(),     torch.nn.MaxPool2d(kernel_size=2), ) ############################################################################  ############################################################################ # Define a sequential network that flattens the features and compiles the # results into probabilities for each digit. results_net = torch.nn.Sequential(     openml_pytorch.layers.Functional(function=torch.Tensor.reshape,                                                 shape=(-1, 4 * 4 * 64)),     torch.nn.Linear(in_features=4 * 4 * 64, out_features=256),     torch.nn.LeakyReLU(),     torch.nn.Dropout(),     torch.nn.Linear(in_features=256, out_features=10), ) ############################################################################ # openml.config.apikey = 'key'  ############################################################################ # The main network, composed of the above specified networks. model = torch.nn.Sequential(     processing_net,     features_net,     results_net ) ############################################################################  In\u00a0[\u00a0]: Copied! <pre>data_module = OpenMLDataModule(\n    type_of_data=\"dataframe\",\n    filename_col=\"class\",\n    target_mode=\"categorical\",\n)\n</pre> data_module = OpenMLDataModule(     type_of_data=\"dataframe\",     filename_col=\"class\",     target_mode=\"categorical\", ) In\u00a0[\u00a0]: Copied! <pre>trainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 1,\n    callbacks=[],\n)\nopenml_pytorch.config.trainer = trainer\n</pre>  trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 1,     callbacks=[], ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre># Download the OpenML task for the mnist 784 dataset.\ntask = openml.tasks.get_task(3573)\n</pre> # Download the OpenML task for the mnist 784 dataset. task = openml.tasks.get_task(3573) In\u00a0[\u00a0]: Copied! <pre>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr() In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish()"},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#sequential-classification","title":"Sequential classification\u00b6","text":"<ul> <li>Sequential classification of a tabular MNIST dataset (Task 3573) using a simple neural network.</li> </ul>"},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the <code>target_col</code> is correctly set.</li> </ul>"},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#download-the-task","title":"Download the task\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"pytorch/Examples/Sequential%20Classification%20Task/#publish-the-run-to-openml","title":"Publish the run to OpenML\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/","title":"Tabular classification","text":"In\u00a0[\u00a0]: Copied! <pre>import torch.nn\nimport torch.optim\n\nimport openml\nimport openml_pytorch\nimport openml_pytorch.layers\nimport openml_pytorch.config\nimport logging\n\n\n############################################################################\n# Enable logging in order to observe the progress while running the example.\nopenml.config.logger.setLevel(logging.DEBUG)\nopenml_pytorch.config.logger.setLevel(logging.DEBUG)\n############################################################################\n</pre>  import torch.nn import torch.optim  import openml import openml_pytorch import openml_pytorch.layers import openml_pytorch.config import logging   ############################################################################ # Enable logging in order to observe the progress while running the example. openml.config.logger.setLevel(logging.DEBUG) openml_pytorch.config.logger.setLevel(logging.DEBUG) ############################################################################ In\u00a0[\u00a0]: Copied! <pre>from openml_pytorch.trainer import OpenMLTrainerModule\nfrom openml_pytorch.trainer import OpenMLDataModule\nfrom openml_pytorch.trainer import Callback\n</pre> from openml_pytorch.trainer import OpenMLTrainerModule from openml_pytorch.trainer import OpenMLDataModule from openml_pytorch.trainer import Callback In\u00a0[\u00a0]: Copied! <pre>class TabularClassificationmodel(torch.nn.Module):\n    def __init__(self, input_size, output_size):\n        super(TabularClassificationmodel, self).__init__()\n        self.fc1 = torch.nn.Linear(input_size, 128)\n        self.fc2 = torch.nn.Linear(128, 64)\n        self.fc3 = torch.nn.Linear(64, output_size)\n        self.relu = torch.nn.ReLU()\n        self.softmax = torch.nn.Softmax(dim=1)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.relu(x)\n        x = self.fc3(x)\n        x = self.softmax(x)\n        return x\n</pre> class TabularClassificationmodel(torch.nn.Module):     def __init__(self, input_size, output_size):         super(TabularClassificationmodel, self).__init__()         self.fc1 = torch.nn.Linear(input_size, 128)         self.fc2 = torch.nn.Linear(128, 64)         self.fc3 = torch.nn.Linear(64, output_size)         self.relu = torch.nn.ReLU()         self.softmax = torch.nn.Softmax(dim=1)      def forward(self, x):         x = self.fc1(x)         x = self.relu(x)         x = self.fc2(x)         x = self.relu(x)         x = self.fc3(x)         x = self.softmax(x)         return x In\u00a0[\u00a0]: Copied! <pre>model = TabularClassificationmodel(20, 2)\n</pre> model = TabularClassificationmodel(20, 2) In\u00a0[\u00a0]: Copied! <pre># supervised credit-g classification\ntask = openml.tasks.get_task(31)\n</pre> # supervised credit-g classification task = openml.tasks.get_task(31) In\u00a0[\u00a0]: Copied! <pre>data_module = OpenMLDataModule(\n    type_of_data=\"dataframe\",\n    target_column=\"class\",\n    target_mode=\"categorical\",\n)\n</pre> data_module = OpenMLDataModule(     type_of_data=\"dataframe\",     target_column=\"class\",     target_mode=\"categorical\", ) In\u00a0[\u00a0]: Copied! <pre>trainer = OpenMLTrainerModule(\n    data_module=data_module,\n    verbose = True,\n    epoch_count = 5,\n)\nopenml_pytorch.config.trainer = trainer\n</pre>  trainer = OpenMLTrainerModule(     data_module=data_module,     verbose = True,     epoch_count = 5, ) openml_pytorch.config.trainer = trainer In\u00a0[\u00a0]: Copied! <pre>run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\n</pre> run = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False) In\u00a0[\u00a0]: Copied! <pre>run.publish()\n</pre> run.publish() In\u00a0[\u00a0]: Copied! <pre># openml.config.apikey = ''\n</pre> # openml.config.apikey = '' In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_loss()\n</pre> trainer.runner.cbs[1].plot_loss() In\u00a0[\u00a0]: Copied! <pre>trainer.runner.cbs[1].plot_lr()\n</pre> trainer.runner.cbs[1].plot_lr()"},{"location":"pytorch/Examples/Tabular%20Classification/#tabular-classification","title":"Tabular classification\u00b6","text":"<ul> <li>Supervised credit-g classification</li> </ul>"},{"location":"pytorch/Examples/Tabular%20Classification/#define-the-model","title":"Define the Model\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#configure-the-data-module","title":"Configure the Data Module\u00b6","text":"<ul> <li>Make sure the <code>target_col</code> is correctly set.</li> </ul>"},{"location":"pytorch/Examples/Tabular%20Classification/#configure-the-trainer-module","title":"Configure the Trainer Module\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#run-the-model-on-the-task","title":"Run the model on the task\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#view-loss","title":"View loss\u00b6","text":""},{"location":"pytorch/Examples/Tabular%20Classification/#view-learning-rate","title":"View learning rate\u00b6","text":""},{"location":"r/","title":"mlr3oml","text":"<p>Package website: release | dev</p> <p>OpenML integration to the mlr3 ecosystem.</p> <p> </p>"},{"location":"r/#what-is-mlr3oml","title":"What is <code>mlr3oml</code>?","text":"<p>OpenML is an open-source platform that facilitates the sharing and dissemination of machine learning research data. All entities on the platform have unique identifiers and standardized (meta)data that can be accessed via an open-access REST API or the web interface. <code>mlr3oml</code> allows to work with the REST API through R and integrates OpenML with the <code>mlr3</code> ecosystem. Note that some upload options are currently not supported, use the OpenML package package for this.</p> <p>As a brief demo, we show how to access an OpenML task, convert it to an <code>mlr3::Task</code> and associated <code>mlr3::Resampling</code>, and conduct a simple resample experiment.</p> <pre><code>library(mlr3oml)\nlibrary(mlr3)\n\n# Download and print the OpenML task with ID 145953\noml_task = otsk(145953)\noml_task\n</code></pre> <pre><code>## &lt;OMLTask:145953&gt;\n##  * Type: Supervised Classification\n##  * Data: kr-vs-kp (id: 3; dim: 3196x37)\n##  * Target: class\n##  * Estimation: crossvalidation (id: 1; repeats: 1, folds: 10)\n</code></pre> <pre><code># Access the OpenML data object on which the task is built\noml_task$data\n</code></pre> <pre><code>## &lt;OMLData:3:kr-vs-kp&gt; (3196x37)\n##  * Default target: class\n</code></pre> <pre><code># Convert the OpenML task to an mlr3 task and resampling\ntask = as_task(oml_task)\nresampling = as_resampling(oml_task)\n\n# Conduct a simple resample experiment\nrr = resample(task, lrn(\"classif.rpart\"), resampling)\nrr$aggregate()\n</code></pre> <pre><code>## classif.ce \n##  0.0319181\n</code></pre> <p>Besides working with objects with known IDs, data of interest can also be queried using listing functions. Below, we search for datasets with 10 - 20 features, 100 to 10000 observations and 2 classes.</p> <pre><code>odatasets = list_oml_data(\n  number_features = c(10, 20),\n  number_instances = c(100, 10000),\n  number_classes = 2\n)\n\nhead(odatasets[, c(\"data_id\", \"name\")])\n</code></pre> <pre><code>##    data_id            name\n## 1:      13   breast-cancer\n## 2:      15        breast-w\n## 3:      29 credit-approval\n## 4:      49         heart-c\n## 5:      50     tic-tac-toe\n## 6:      51         heart-h\n</code></pre> <p>To retrieve individual datasets, you can use <code>odt</code> and either manually construct a new <code>Task</code> object using <code>as_task()</code> or use it <code>data.table</code> format.</p> <pre><code>odataset = odt(29)\n\n# Dataset as data.table\nstr(odataset$data)\n</code></pre> <pre><code>## Classes 'data.table' and 'data.frame':   690 obs. of  16 variables:\n##  $ A1   : Factor w/ 2 levels \"b\",\"a\": 1 2 2 1 1 1 1 2 1 1 ...\n##  $ A2   : num  30.8 58.7 24.5 27.8 20.2 ...\n##  $ A3   : num  0 4.46 0.5 1.54 5.62 ...\n##  $ A4   : Factor w/ 4 levels \"u\",\"y\",\"l\",\"t\": 1 1 1 1 1 1 1 1 2 2 ...\n##  $ A5   : Factor w/ 3 levels \"g\",\"p\",\"gg\": 1 1 1 1 1 1 1 1 2 2 ...\n##  $ A6   : Factor w/ 14 levels \"c\",\"d\",\"cc\",\"i\",..: 10 9 9 10 10 7 8 3 6 10 ...\n##  $ A7   : Factor w/ 9 levels \"v\",\"h\",\"bb\",\"j\",..: 1 2 2 1 1 1 2 1 2 1 ...\n##  $ A8   : num  1.25 3.04 1.5 3.75 1.71 ...\n##  $ A9   : Factor w/ 2 levels \"t\",\"f\": 1 1 1 1 1 1 1 1 1 1 ...\n##  $ A10  : Factor w/ 2 levels \"t\",\"f\": 1 1 2 1 2 2 2 2 2 2 ...\n##  $ A11  : int  1 6 0 5 0 0 0 0 0 0 ...\n##  $ A12  : Factor w/ 2 levels \"t\",\"f\": 2 2 2 1 2 1 1 2 2 1 ...\n##  $ A13  : Factor w/ 3 levels \"g\",\"p\",\"s\": 1 1 1 1 3 1 1 1 1 1 ...\n##  $ A14  : int  202 43 280 100 120 360 164 80 180 52 ...\n##  $ A15  : int  0 560 824 3 0 0 31285 1349 314 1442 ...\n##  $ class: Factor w/ 2 levels \"+\",\"-\": 1 1 1 1 1 1 1 1 1 1 ...\n##  - attr(*, \".internal.selfref\")=&lt;externalptr&gt;\n</code></pre> <pre><code># Creating a new task\notask = as_task(odataset)\notask\n</code></pre> <pre><code>## &lt;TaskClassif:credit-approval&gt; (690 x 16)\n## * Target: class\n## * Properties: twoclass\n## * Features (15):\n##   - fct (9): A1, A10, A12, A13, A4, A5, A6, A7, A9\n##   - int (3): A11, A14, A15\n##   - dbl (3): A2, A3, A8\n</code></pre>"},{"location":"r/#feature-overview","title":"Feature Overview","text":"<ul> <li>Datasets, tasks, flows, runs, and collections can be downloaded from   OpenML and are represented as <code>R6</code> classes.</li> <li>OpenML objects can be easily converted to the corresponding <code>mlr3</code>   counterpart.</li> <li>Filtering of OpenML objects can be achieved using listing functions.</li> <li>Downloaded objects can be cached by setting the <code>mlr3oml.cache</code>   option.</li> <li>Both the <code>arff</code> and <code>parquet</code> filetype for datasets are supported.</li> <li>You can upload datasets, tasks, and collections to OpenML.</li> </ul>"},{"location":"r/#documentation","title":"Documentation","text":"<ul> <li>Start by reading the Large-Scale Benchmarking   chapter   from the <code>mlr3</code> book.</li> <li>The package website contains a   getting started guide.</li> <li>The OpenML API documentation is also a   good resource.</li> </ul>"},{"location":"r/#bugs-questions-feedback","title":"Bugs, Questions, Feedback","text":"<p>mlr3oml is a free and open source software project that encourages participation and feedback. If you have any issues, questions, suggestions or feedback, please do not hesitate to open an \u201cissue\u201d about it on the GitHub page!</p> <p>In case of problems / bugs, it is often helpful if you provide a \u201cminimum working example\u201d that showcases the behaviour (but don\u2019t worry about this if the bug is obvious).</p>"},{"location":"reference/","title":"Index","text":"<p>The OpenML module implements a python interface to <code>OpenML &lt;https://www.openml.org&gt;</code>_, a collaborative platform for machine learning. OpenML can be used to</p> <ul> <li>store, download and analyze datasets</li> <li>make experiments and their results (e.g. models, predictions)   accesible and reproducible for everybody</li> <li>analyze experiments (uploaded by you and other collaborators) and conduct   meta studies</li> </ul> <p>In particular, this module implements a python interface for the <code>OpenML REST API &lt;https://www.openml.org/guide#!rest_services&gt;</code>_ (<code>REST on wikipedia &lt;https://en.wikipedia.org/wiki/Representational_state_transfer&gt;</code>_).</p>"},{"location":"reference/#openml.OpenMLBenchmarkSuite","title":"<code>OpenMLBenchmarkSuite</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).</p> <p>It contains the following information: name, id, description, creation date, creator id and the task ids.</p> <p>According to this list of task ids, the suite object receives a list of OpenML object ids (datasets).</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLBenchmarkSuite(BaseStudy):\n    \"\"\"\n    An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and the task ids.\n\n    According to this list of task ids, the suite object receives a list of\n    OpenML object ids (datasets).\n\n    Parameters\n    ----------\n    suite_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        suite_id: int | None,\n        alias: str | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n    ):\n        super().__init__(\n            study_id=suite_id,\n            alias=alias,\n            main_entity_type=\"task\",\n            benchmark_suite=None,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=None,\n            runs=None,\n            setups=None,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLClassificationTask","title":"<code>OpenMLClassificationTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Classification task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the Classification task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the Classification task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target variable.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the Classification task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the Classification task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the Classification task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Classification task (if it already exists on OpenML).</p> <code>None</code> <code>class_labels</code> <code>List of str</code> <p>A list of class labels (for classification tasks).</p> <code>None</code> <code>cost_matrix</code> <code>array</code> <p>A cost matrix (for classification tasks).</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClassificationTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Classification task type.\n    task_type : str\n        Name of the Classification task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the Classification task.\n    target_name : str\n        Name of the target variable.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the Classification task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the Classification task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure.\n    data_splits_url : str, default=None\n        URL of the data splits for the Classification task.\n    task_id : Union[int, None]\n        ID of the Classification task (if it already exists on OpenML).\n    class_labels : List of str, default=None\n        A list of class labels (for classification tasks).\n    cost_matrix : array, default=None\n        A cost matrix (for classification tasks).\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n        self.class_labels = class_labels\n        self.cost_matrix = cost_matrix\n\n        if cost_matrix is not None:\n            raise NotImplementedError(\"Costmatrix\")\n</code></pre>"},{"location":"reference/#openml.OpenMLClusteringTask","title":"<code>OpenMLClusteringTask</code>","text":"<p>               Bases: <code>OpenMLTask</code></p> <p>OpenML Clustering object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML clustering task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML clustering task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset used in clustering the task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML clustering task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure used in the clustering task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the clustering task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the clustering task.</p> <code>None</code> <code>target_name</code> <code>str</code> <p>Name of the target feature (class) that is not part of the feature set for the clustering task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClusteringTask(OpenMLTask):\n    \"\"\"OpenML Clustering object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML clustering task.\n    task_type : str\n        Task type of the OpenML clustering task.\n    data_set_id : int\n        ID of the OpenML dataset used in clustering the task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    task_id : Union[int, None]\n        ID of the OpenML clustering task.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure used in the clustering task.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the clustering task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the clustering task.\n    target_name : str, default=None\n        Name of the target feature (class) that is not part of the\n        feature set for the clustering task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 17,\n        task_id: int | None = None,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        evaluation_measure: str | None = None,\n        target_name: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            evaluation_measure=evaluation_measure,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\"] = \"array\",\n    ) -&gt; np.ndarray | scipy.sparse.spmatrix:\n        ...\n\n    @overload\n    def get_X(self, dataset_format: Literal[\"dataframe\"]) -&gt; pd.DataFrame:\n        ...\n\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n    ) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        dataset = self.get_dataset()\n        data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n        return data\n\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        # Right now, it is not supported as a feature.\n        # Uncomment if it is supported on the server\n        # in the future.\n        # https://github.com/openml/OpenML/issues/925\n        \"\"\"\n        task_dict = task_container['oml:task_inputs']\n        if self.target_name is not None:\n            task_dict['oml:input'].append(\n                OrderedDict([\n                    ('@name', 'target_feature'),\n                    ('#text', self.target_name)\n                ])\n            )\n        \"\"\"\n        return super()._to_dict()\n</code></pre>"},{"location":"reference/#openml.OpenMLClusteringTask.get_X","title":"<code>get_X(dataset_format='array')</code>","text":"<pre><code>get_X(dataset_format: Literal['array'] = 'array') -&gt; np.ndarray | scipy.sparse.spmatrix\n</code></pre><pre><code>get_X(dataset_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X(\n    self,\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    dataset = self.get_dataset()\n    data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n    return data\n</code></pre>"},{"location":"reference/#openml.OpenMLDataFeature","title":"<code>OpenMLDataFeature</code>","text":"<p>Data Feature (a.k.a. Attribute) object.</p> <p>Parameters:</p> Name Type Description Default <code>index</code> <code>int</code> <p>The index of this feature</p> required <code>name</code> <code>str</code> <p>Name of the feature</p> required <code>data_type</code> <code>str</code> <p>can be nominal, numeric, string, date (corresponds to arff)</p> required <code>nominal_values</code> <code>list(str)</code> <p>list of the possible values, in case of nominal attribute</p> required <code>number_missing_values</code> <code>int</code> <p>Number of rows that have a missing value for this feature.</p> required <code>ontologies</code> <code>list(str)</code> <p>list of ontologies attached to this feature. An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided.</p> <code>None</code> Source code in <code>openml/datasets/data_feature.py</code> <pre><code>class OpenMLDataFeature:\n    \"\"\"\n    Data Feature (a.k.a. Attribute) object.\n\n    Parameters\n    ----------\n    index : int\n        The index of this feature\n    name : str\n        Name of the feature\n    data_type : str\n        can be nominal, numeric, string, date (corresponds to arff)\n    nominal_values : list(str)\n        list of the possible values, in case of nominal attribute\n    number_missing_values : int\n        Number of rows that have a missing value for this feature.\n    ontologies : list(str)\n        list of ontologies attached to this feature. An ontology describes the\n        concept that are described in a feature. An ontology is defined by an\n        URL where the information is provided.\n    \"\"\"\n\n    LEGAL_DATA_TYPES: ClassVar[Sequence[str]] = [\"nominal\", \"numeric\", \"string\", \"date\"]\n\n    def __init__(  # noqa: PLR0913\n        self,\n        index: int,\n        name: str,\n        data_type: str,\n        nominal_values: list[str],\n        number_missing_values: int,\n        ontologies: list[str] | None = None,\n    ):\n        if not isinstance(index, int):\n            raise TypeError(f\"Index must be `int` but is {type(index)}\")\n\n        if data_type not in self.LEGAL_DATA_TYPES:\n            raise ValueError(\n                f\"data type should be in {self.LEGAL_DATA_TYPES!s}, found: {data_type}\",\n            )\n\n        if data_type == \"nominal\":\n            if nominal_values is None:\n                raise TypeError(\n                    \"Dataset features require attribute `nominal_values` for nominal \"\n                    \"feature type.\",\n                )\n\n            if not isinstance(nominal_values, list):\n                raise TypeError(\n                    \"Argument `nominal_values` is of wrong datatype, should be list, \"\n                    f\"but is {type(nominal_values)}\",\n                )\n        elif nominal_values is not None:\n            raise TypeError(\"Argument `nominal_values` must be None for non-nominal feature.\")\n\n        if not isinstance(number_missing_values, int):\n            msg = f\"number_missing_values must be int but is {type(number_missing_values)}\"\n            raise TypeError(msg)\n\n        self.index = index\n        self.name = str(name)\n        self.data_type = str(data_type)\n        self.nominal_values = nominal_values\n        self.number_missing_values = number_missing_values\n        self.ontologies = ontologies\n\n    def __repr__(self) -&gt; str:\n        return \"[%d - %s (%s)]\" % (self.index, self.name, self.data_type)\n\n    def __eq__(self, other: Any) -&gt; bool:\n        return isinstance(other, OpenMLDataFeature) and self.__dict__ == other.__dict__\n\n    def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -&gt; None:  # noqa: FBT001, ARG002\n        pp.text(str(self))\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset","title":"<code>OpenMLDataset</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>Dataset object.</p> <p>Allows fetching and uploading datasets to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>data_format</code> <code>str</code> <p>Format of the dataset which can be either 'arff' or 'sparse_arff'.</p> <code>'arff'</code> <code>cache_format</code> <code>str</code> <p>Format for caching the dataset which can be either 'feather' or 'pickle'.</p> <code>'pickle'</code> <code>dataset_id</code> <code>int</code> <p>Id autogenerated by the server.</p> <code>None</code> <code>version</code> <code>int</code> <p>Version of this dataset. '1' for original version. Auto-incremented by server.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>The date-time when the dataset was uploaded, generated by server.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>licence</code> <code>str</code> <p>License of the data.</p> <code>None</code> <code>url</code> <code>str</code> <p>Valid URL, points to actual data file. The file can be on the OpenML server or another dataset repository.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>version_label</code> <code>str</code> <p>Version label provided by user. Can be a date, hash, or some other type of id.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>tag</code> <code>str</code> <p>Tags, describing the algorithms.</p> <code>None</code> <code>visibility</code> <code>str</code> <p>Who can see the dataset. Typical values: 'Everyone','All my friends','Only me'. Can also be any of the user's circles.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <code>md5_checksum</code> <code>str</code> <p>MD5 checksum to check if the dataset is downloaded without corruption.</p> <code>None</code> <code>data_file</code> <code>str</code> <p>Path to where the dataset is located.</p> <code>None</code> <code>features_file</code> <code>dict</code> <p>A dictionary of dataset features, which maps a feature index to a OpenMLDataFeature.</p> <code>None</code> <code>qualities_file</code> <code>dict</code> <p>A dictionary of dataset qualities, which maps a quality name to a quality value.</p> <code>None</code> <code>dataset</code> <code>str | None</code> <p>Serialized arff dataset string.</p> <code>None</code> <code>parquet_url</code> <code>str | None</code> <p>This is the URL to the storage location where the dataset files are hosted. This can be a MinIO bucket URL. If specified, the data will be accessed from this URL when reading the files.</p> <code>None</code> <code>parquet_file</code> <code>str | None</code> <p>Path to the local file.</p> <code>None</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>class OpenMLDataset(OpenMLBase):\n    \"\"\"Dataset object.\n\n    Allows fetching and uploading datasets to OpenML.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    data_format : str\n        Format of the dataset which can be either 'arff' or 'sparse_arff'.\n    cache_format : str\n        Format for caching the dataset which can be either 'feather' or 'pickle'.\n    dataset_id : int, optional\n        Id autogenerated by the server.\n    version : int, optional\n        Version of this dataset. '1' for original version.\n        Auto-incremented by server.\n    creator : str, optional\n        The person who created the dataset.\n    contributor : str, optional\n        People who contributed to the current version of the dataset.\n    collection_date : str, optional\n        The date the data was originally collected, given by the uploader.\n    upload_date : str, optional\n        The date-time when the dataset was uploaded, generated by server.\n    language : str, optional\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str, optional\n        License of the data.\n    url : str, optional\n        Valid URL, points to actual data file.\n        The file can be on the OpenML server or another dataset repository.\n    default_target_attribute : str, optional\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column,\n        if present in the dataset.\n    ignore_attribute : str | list, optional\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    version_label : str, optional\n        Version label provided by user.\n        Can be a date, hash, or some other type of id.\n    citation : str, optional\n        Reference(s) that should be cited when building on this data.\n    tag : str, optional\n        Tags, describing the algorithms.\n    visibility : str, optional\n        Who can see the dataset.\n        Typical values: 'Everyone','All my friends','Only me'.\n        Can also be any of the user's circles.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n    md5_checksum : str, optional\n        MD5 checksum to check if the dataset is downloaded without corruption.\n    data_file : str, optional\n        Path to where the dataset is located.\n    features_file : dict, optional\n        A dictionary of dataset features,\n        which maps a feature index to a OpenMLDataFeature.\n    qualities_file : dict, optional\n        A dictionary of dataset qualities,\n        which maps a quality name to a quality value.\n    dataset: string, optional\n        Serialized arff dataset string.\n    parquet_url: string, optional\n        This is the URL to the storage location where the dataset files are hosted.\n        This can be a MinIO bucket URL. If specified, the data will be accessed\n        from this URL when reading the files.\n    parquet_file: string, optional\n        Path to the local file.\n    \"\"\"\n\n    def __init__(  # noqa: C901, PLR0912, PLR0913, PLR0915\n        self,\n        name: str,\n        description: str | None,\n        data_format: Literal[\"arff\", \"sparse_arff\"] = \"arff\",\n        cache_format: Literal[\"feather\", \"pickle\"] = \"pickle\",\n        dataset_id: int | None = None,\n        version: int | None = None,\n        creator: str | None = None,\n        contributor: str | None = None,\n        collection_date: str | None = None,\n        upload_date: str | None = None,\n        language: str | None = None,\n        licence: str | None = None,\n        url: str | None = None,\n        default_target_attribute: str | None = None,\n        row_id_attribute: str | None = None,\n        ignore_attribute: str | list[str] | None = None,\n        version_label: str | None = None,\n        citation: str | None = None,\n        tag: str | None = None,\n        visibility: str | None = None,\n        original_data_url: str | None = None,\n        paper_url: str | None = None,\n        update_comment: str | None = None,\n        md5_checksum: str | None = None,\n        data_file: str | None = None,\n        features_file: str | None = None,\n        qualities_file: str | None = None,\n        dataset: str | None = None,\n        parquet_url: str | None = None,\n        parquet_file: str | None = None,\n    ):\n        if cache_format not in [\"feather\", \"pickle\"]:\n            raise ValueError(\n                \"cache_format must be one of 'feather' or 'pickle. \"\n                f\"Invalid format specified: {cache_format}\",\n            )\n\n        def find_invalid_characters(string: str, pattern: str) -&gt; str:\n            invalid_chars = set()\n            regex = re.compile(pattern)\n            for char in string:\n                if not regex.match(char):\n                    invalid_chars.add(char)\n            return \",\".join(\n                [f\"'{char}'\" if char != \"'\" else f'\"{char}\"' for char in invalid_chars],\n            )\n\n        if dataset_id is None:\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if description and not re.match(pattern, description):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(description, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in description: {description}\",\n                )\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if citation and not re.match(pattern, citation):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(citation, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in citation: {citation}\",\n                )\n            pattern = \"^[a-zA-Z0-9_\\\\-\\\\.\\\\(\\\\),]+$\"\n            if not re.match(pattern, name):\n                # regex given by server in error message\n                invalid_characters = find_invalid_characters(name, pattern)\n                raise ValueError(f\"Invalid symbols {invalid_characters} in name: {name}\")\n\n        self.ignore_attribute: list[str] | None = None\n        if isinstance(ignore_attribute, str):\n            self.ignore_attribute = [ignore_attribute]\n        elif isinstance(ignore_attribute, list) or ignore_attribute is None:\n            self.ignore_attribute = ignore_attribute\n        else:\n            raise ValueError(\"Wrong data type for ignore_attribute. Should be list.\")\n\n        # TODO add function to check if the name is casual_string128\n        # Attributes received by querying the RESTful API\n        self.dataset_id = int(dataset_id) if dataset_id is not None else None\n        self.name = name\n        self.version = int(version) if version is not None else None\n        self.description = description\n        self.cache_format = cache_format\n        # Has to be called format, otherwise there will be an XML upload error\n        self.format = data_format\n        self.creator = creator\n        self.contributor = contributor\n        self.collection_date = collection_date\n        self.upload_date = upload_date\n        self.language = language\n        self.licence = licence\n        self.url = url\n        self.default_target_attribute = default_target_attribute\n        self.row_id_attribute = row_id_attribute\n\n        self.version_label = version_label\n        self.citation = citation\n        self.tag = tag\n        self.visibility = visibility\n        self.original_data_url = original_data_url\n        self.paper_url = paper_url\n        self.update_comment = update_comment\n        self.md5_checksum = md5_checksum\n        self.data_file = data_file\n        self.parquet_file = parquet_file\n        self._dataset = dataset\n        self._parquet_url = parquet_url\n\n        self._features: dict[int, OpenMLDataFeature] | None = None\n        self._qualities: dict[str, float] | None = None\n        self._no_qualities_found = False\n\n        if features_file is not None:\n            self._features = _read_features(Path(features_file))\n\n        # \"\" was the old default value by `get_dataset` and maybe still used by some\n        if qualities_file == \"\":\n            # TODO(0.15): to switch to \"qualities_file is not None\" below and remove warning\n            warnings.warn(\n                \"Starting from Version 0.15 `qualities_file` must be None and not an empty string \"\n                \"to avoid reading the qualities from file. Set `qualities_file` to None to avoid \"\n                \"this warning.\",\n                FutureWarning,\n                stacklevel=2,\n            )\n            qualities_file = None\n\n        if qualities_file is not None:\n            self._qualities = _read_qualities(Path(qualities_file))\n\n        if data_file is not None:\n            data_pickle, data_feather, feather_attribute = self._compressed_cache_file_paths(\n                Path(data_file)\n            )\n            self.data_pickle_file = data_pickle if Path(data_pickle).exists() else None\n            self.data_feather_file = data_feather if Path(data_feather).exists() else None\n            self.feather_attribute_file = feather_attribute if Path(feather_attribute) else None\n        else:\n            self.data_pickle_file = None\n            self.data_feather_file = None\n            self.feather_attribute_file = None\n\n    @property\n    def features(self) -&gt; dict[int, OpenMLDataFeature]:\n        \"\"\"Get the features of this dataset.\"\"\"\n        if self._features is None:\n            # TODO(eddiebergman): These should return a value so we can set it to be not None\n            self._load_features()\n\n        assert self._features is not None\n        return self._features\n\n    @property\n    def qualities(self) -&gt; dict[str, float] | None:\n        \"\"\"Get the qualities of this dataset.\"\"\"\n        # TODO(eddiebergman): Better docstring, I don't know what qualities means\n\n        # We have to check `_no_qualities_found` as there might not be qualities for a dataset\n        if self._qualities is None and (not self._no_qualities_found):\n            self._load_qualities()\n\n        return self._qualities\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Get the dataset numeric id.\"\"\"\n        return self.dataset_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Obtain number of features in accordance with lazy loading.\n        n_features: int | None = None\n        if self._qualities is not None and self._qualities[\"NumberOfFeatures\"] is not None:\n            n_features = int(self._qualities[\"NumberOfFeatures\"])\n        elif self._features is not None:\n            n_features = len(self._features)\n\n        fields: dict[str, int | str | None] = {\n            \"Name\": self.name,\n            \"Version\": self.version,\n            \"Format\": self.format,\n            \"Licence\": self.licence,\n            \"Download URL\": self.url,\n            \"Data file\": str(self.data_file) if self.data_file is not None else None,\n            \"Pickle file\": (\n                str(self.data_pickle_file) if self.data_pickle_file is not None else None\n            ),\n            \"# of features\": n_features,\n        }\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.dataset_id is not None:\n            fields[\"OpenML URL\"] = self.openml_url\n        if self._qualities is not None and self._qualities[\"NumberOfInstances\"] is not None:\n            fields[\"# of instances\"] = int(self._qualities[\"NumberOfInstances\"])\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Name\",\n            \"Version\",\n            \"Format\",\n            \"Upload Date\",\n            \"Licence\",\n            \"Download URL\",\n            \"OpenML URL\",\n            \"Data File\",\n            \"Pickle File\",\n            \"# of features\",\n            \"# of instances\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if not isinstance(other, OpenMLDataset):\n            return False\n\n        server_fields = {\n            \"dataset_id\",\n            \"version\",\n            \"upload_date\",\n            \"url\",\n            \"dataset\",\n            \"data_file\",\n        }\n\n        # check that common keys and values are identical\n        self_keys = set(self.__dict__.keys()) - server_fields\n        other_keys = set(other.__dict__.keys()) - server_fields\n        return self_keys == other_keys and all(\n            self.__dict__[key] == other.__dict__[key] for key in self_keys\n        )\n\n    def _download_data(self) -&gt; None:\n        \"\"\"Download ARFF data file to standard cache directory. Set `self.data_file`.\"\"\"\n        # import required here to avoid circular import.\n        from .functions import _get_dataset_arff, _get_dataset_parquet\n\n        self.data_file = str(_get_dataset_arff(self))\n        if self._parquet_url is not None:\n            self.parquet_file = str(_get_dataset_parquet(self))\n\n    def _get_arff(self, format: str) -&gt; dict:  # noqa: A002\n        \"\"\"Read ARFF file and return decoded arff.\n\n        Reads the file referenced in self.data_file.\n\n        Parameters\n        ----------\n        format : str\n            Format of the ARFF file.\n            Must be one of 'arff' or 'sparse_arff' or a string that will be either of those\n            when converted to lower case.\n\n\n\n        Returns\n        -------\n        dict\n            Decoded arff.\n\n        \"\"\"\n        # TODO: add a partial read method which only returns the attribute\n        # headers of the corresponding .arff file!\n        import struct\n\n        filename = self.data_file\n        assert filename is not None\n        filepath = Path(filename)\n\n        bits = 8 * struct.calcsize(\"P\")\n\n        # Files can be considered too large on a 32-bit system,\n        # if it exceeds 120mb (slightly more than covtype dataset size)\n        # This number is somewhat arbitrary.\n        if bits != 64:\n            MB_120 = 120_000_000\n            file_size = filepath.stat().st_size\n            if file_size &gt; MB_120:\n                raise NotImplementedError(\n                    f\"File {filename} too big for {file_size}-bit system ({bits} bytes).\",\n                )\n\n        if format.lower() == \"arff\":\n            return_type = arff.DENSE\n        elif format.lower() == \"sparse_arff\":\n            return_type = arff.COO\n        else:\n            raise ValueError(f\"Unknown data format {format}\")\n\n        def decode_arff(fh: Any) -&gt; dict:\n            decoder = arff.ArffDecoder()\n            return decoder.decode(fh, encode_nominal=True, return_type=return_type)  # type: ignore\n\n        if filepath.suffix.endswith(\".gz\"):\n            with gzip.open(filename) as zipfile:\n                return decode_arff(zipfile)\n        else:\n            with filepath.open(encoding=\"utf8\") as fh:\n                return decode_arff(fh)\n\n    def _parse_data_from_arff(  # noqa: C901, PLR0912, PLR0915\n        self,\n        arff_file_path: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Parse all required data from arff file.\n\n        Parameters\n        ----------\n        arff_file_path : str\n            Path to the file on disk.\n\n        Returns\n        -------\n        Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]\n            DataFrame or csr_matrix: dataset\n            List[bool]: List indicating which columns contain categorical variables.\n            List[str]: List of column names.\n        \"\"\"\n        try:\n            data = self._get_arff(self.format)\n        except OSError as e:\n            logger.critical(\n                f\"Please check that the data file {arff_file_path} is \" \"there and can be read.\",\n            )\n            raise e\n\n        ARFF_DTYPES_TO_PD_DTYPE = {\n            \"INTEGER\": \"integer\",\n            \"REAL\": \"floating\",\n            \"NUMERIC\": \"floating\",\n            \"STRING\": \"string\",\n        }\n        attribute_dtype = {}\n        attribute_names = []\n        categories_names = {}\n        categorical = []\n        for name, type_ in data[\"attributes\"]:\n            # if the feature is nominal and a sparse matrix is\n            # requested, the categories need to be numeric\n            if isinstance(type_, list) and self.format.lower() == \"sparse_arff\":\n                try:\n                    # checks if the strings which should be the class labels\n                    # can be encoded into integers\n                    pd.factorize(type_)[0]\n                except ValueError as e:\n                    raise ValueError(\n                        \"Categorical data needs to be numeric when using sparse ARFF.\"\n                    ) from e\n\n            # string can only be supported with pandas DataFrame\n            elif type_ == \"STRING\" and self.format.lower() == \"sparse_arff\":\n                raise ValueError(\"Dataset containing strings is not supported with sparse ARFF.\")\n\n            # infer the dtype from the ARFF header\n            if isinstance(type_, list):\n                categorical.append(True)\n                categories_names[name] = type_\n                if len(type_) == 2:\n                    type_norm = [cat.lower().capitalize() for cat in type_]\n                    if {\"True\", \"False\"} == set(type_norm):\n                        categories_names[name] = [cat == \"True\" for cat in type_norm]\n                        attribute_dtype[name] = \"boolean\"\n                    else:\n                        attribute_dtype[name] = \"categorical\"\n                else:\n                    attribute_dtype[name] = \"categorical\"\n            else:\n                categorical.append(False)\n                attribute_dtype[name] = ARFF_DTYPES_TO_PD_DTYPE[type_]\n            attribute_names.append(name)\n\n        if self.format.lower() == \"sparse_arff\":\n            X = data[\"data\"]\n            X_shape = (max(X[1]) + 1, max(X[2]) + 1)\n            X = scipy.sparse.coo_matrix((X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32)\n            X = X.tocsr()\n        elif self.format.lower() == \"arff\":\n            X = pd.DataFrame(data[\"data\"], columns=attribute_names)\n\n            col = []\n            for column_name in X.columns:\n                if attribute_dtype[column_name] in (\"categorical\", \"boolean\"):\n                    categories = self._unpack_categories(\n                        X[column_name],  # type: ignore\n                        categories_names[column_name],\n                    )\n                    col.append(categories)\n                elif attribute_dtype[column_name] in (\"floating\", \"integer\"):\n                    X_col = X[column_name]\n                    if X_col.min() &gt;= 0 and X_col.max() &lt;= 255:\n                        try:\n                            X_col_uint = X_col.astype(\"uint8\")\n                            if (X_col == X_col_uint).all():\n                                col.append(X_col_uint)\n                                continue\n                        except ValueError:\n                            pass\n                    col.append(X[column_name])\n                else:\n                    col.append(X[column_name])\n            X = pd.concat(col, axis=1)\n        else:\n            raise ValueError(f\"Dataset format '{self.format}' is not a valid format.\")\n\n        return X, categorical, attribute_names  # type: ignore\n\n    def _compressed_cache_file_paths(self, data_file: Path) -&gt; tuple[Path, Path, Path]:\n        data_pickle_file = data_file.with_suffix(\".pkl.py3\")\n        data_feather_file = data_file.with_suffix(\".feather\")\n        feather_attribute_file = data_file.with_suffix(\".feather.attributes.pkl.py3\")\n        return data_pickle_file, data_feather_file, feather_attribute_file\n\n    def _cache_compressed_file_from_file(\n        self,\n        data_file: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Store data from the local file in compressed format.\n\n        If a local parquet file is present it will be used instead of the arff file.\n        Sets cache_format to 'pickle' if data is sparse.\n        \"\"\"\n        (\n            data_pickle_file,\n            data_feather_file,\n            feather_attribute_file,\n        ) = self._compressed_cache_file_paths(data_file)\n\n        if data_file.suffix == \".arff\":\n            data, categorical, attribute_names = self._parse_data_from_arff(data_file)\n        elif data_file.suffix == \".pq\":\n            try:\n                data = pd.read_parquet(data_file)\n            except Exception as e:  # noqa: BLE001\n                raise Exception(f\"File: {data_file}\") from e\n\n            categorical = [data[c].dtype.name == \"category\" for c in data.columns]\n            attribute_names = list(data.columns)\n        else:\n            raise ValueError(f\"Unknown file type for file '{data_file}'.\")\n\n        # Feather format does not work for sparse datasets, so we use pickle for sparse datasets\n        if scipy.sparse.issparse(data):\n            self.cache_format = \"pickle\"\n\n        logger.info(f\"{self.cache_format} write {self.name}\")\n        if self.cache_format == \"feather\":\n            assert isinstance(data, pd.DataFrame)\n\n            data.to_feather(data_feather_file)\n            with open(feather_attribute_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_feather_file = data_feather_file\n            self.feather_attribute_file = feather_attribute_file\n\n        else:\n            with open(data_pickle_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_pickle_file = data_pickle_file\n\n        data_file = data_pickle_file if self.cache_format == \"pickle\" else data_feather_file\n        logger.debug(f\"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}\")\n\n        return data, categorical, attribute_names\n\n    def _load_data(self) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:  # noqa: PLR0912, C901\n        \"\"\"Load data from compressed format or arff. Download data if not present on disk.\"\"\"\n        need_to_create_pickle = self.cache_format == \"pickle\" and self.data_pickle_file is None\n        need_to_create_feather = self.cache_format == \"feather\" and self.data_feather_file is None\n\n        if need_to_create_pickle or need_to_create_feather:\n            if self.data_file is None:\n                self._download_data()\n\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n\n        # helper variable to help identify where errors occur\n        fpath = self.data_feather_file if self.cache_format == \"feather\" else self.data_pickle_file\n        logger.info(f\"{self.cache_format} load data {self.name}\")\n        try:\n            if self.cache_format == \"feather\":\n                assert self.data_feather_file is not None\n                assert self.feather_attribute_file is not None\n\n                data = pd.read_feather(self.data_feather_file)\n                fpath = self.feather_attribute_file\n                with open(self.feather_attribute_file, \"rb\") as fh:  # noqa: PTH123\n                    categorical, attribute_names = pickle.load(fh)  # noqa: S301\n            else:\n                assert self.data_pickle_file is not None\n                with open(self.data_pickle_file, \"rb\") as fh:  # noqa: PTH123\n                    data, categorical, attribute_names = pickle.load(fh)  # noqa: S301\n        except FileNotFoundError as e:\n            raise ValueError(\n                f\"Cannot find file for dataset {self.name} at location '{fpath}'.\"\n            ) from e\n        except (EOFError, ModuleNotFoundError, ValueError, AttributeError) as e:\n            error_message = getattr(e, \"message\", e.args[0])\n            hint = \"\"\n\n            if isinstance(e, EOFError):\n                readable_error = \"Detected a corrupt cache file\"\n            elif isinstance(e, (ModuleNotFoundError, AttributeError)):\n                readable_error = \"Detected likely dependency issues\"\n                hint = (\n                    \"This can happen if the cache was constructed with a different pandas version \"\n                    \"than the one that is used to load the data. See also \"\n                )\n                if isinstance(e, ModuleNotFoundError):\n                    hint += \"https://github.com/openml/openml-python/issues/918. \"\n                elif isinstance(e, AttributeError):\n                    hint += \"https://github.com/openml/openml-python/pull/1121. \"\n\n            elif isinstance(e, ValueError) and \"unsupported pickle protocol\" in e.args[0]:\n                readable_error = \"Encountered unsupported pickle protocol\"\n            else:\n                raise e\n\n            logger.warning(\n                f\"{readable_error} when loading dataset {self.id} from '{fpath}'. \"\n                f\"{hint}\"\n                f\"Error message was: {error_message}. \"\n                \"We will continue loading data from the arff-file, \"\n                \"but this will be much slower for big datasets. \"\n                \"Please manually delete the cache file if you want OpenML-Python \"\n                \"to attempt to reconstruct it.\",\n            )\n            assert self.data_file is not None\n            data, categorical, attribute_names = self._parse_data_from_arff(Path(self.data_file))\n\n        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)\n        if self.cache_format == \"pickle\" and not data_up_to_date:\n            logger.info(\"Updating outdated pickle file.\")\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n        return data, categorical, attribute_names\n\n    # TODO(eddiebergman): Can type this better with overload\n    # TODO(eddiebergman): Could also techinically use scipy.sparse.sparray\n    @staticmethod\n    def _convert_array_format(\n        data: pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix,\n        array_format: Literal[\"array\", \"dataframe\"],\n        attribute_names: list | None = None,\n    ) -&gt; pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix:\n        \"\"\"Convert a dataset to a given array format.\n\n        Converts to numpy array if data is non-sparse.\n        Converts to a sparse dataframe if data is sparse.\n\n        Parameters\n        ----------\n        array_format : str {'array', 'dataframe'}\n            Desired data type of the output\n            - If array_format='array'\n                If data is non-sparse\n                    Converts to numpy-array\n                    Enforces numeric encoding of categorical columns\n                    Missing values are represented as NaN in the numpy-array\n                else returns data as is\n            - If array_format='dataframe'\n                If data is sparse\n                    Works only on sparse data\n                    Converts sparse data to sparse dataframe\n                else returns data as is\n\n        \"\"\"\n        if array_format == \"array\" and not isinstance(data, scipy.sparse.spmatrix):\n            # We encode the categories such that they are integer to be able\n            # to make a conversion to numeric for backward compatibility\n            def _encode_if_category(column: pd.Series | np.ndarray) -&gt; pd.Series | np.ndarray:\n                if column.dtype.name == \"category\":\n                    column = column.cat.codes.astype(np.float32)\n                    mask_nan = column == -1\n                    column[mask_nan] = np.nan\n                return column\n\n            if isinstance(data, pd.DataFrame):\n                columns = {\n                    column_name: _encode_if_category(data.loc[:, column_name])\n                    for column_name in data.columns\n                }\n                data = pd.DataFrame(columns)\n            else:\n                data = _encode_if_category(data)\n\n            try:\n                # TODO(eddiebergman): float32?\n                return_array = np.asarray(data, dtype=np.float32)\n            except ValueError as e:\n                raise PyOpenMLError(\n                    \"PyOpenML cannot handle string when returning numpy\"\n                    ' arrays. Use dataset_format=\"dataframe\".',\n                ) from e\n\n            return return_array\n\n        if array_format == \"dataframe\":\n            if scipy.sparse.issparse(data):\n                data = pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)\n        else:\n            data_type = \"sparse-data\" if scipy.sparse.issparse(data) else \"non-sparse data\"\n            logger.warning(\n                f\"Cannot convert {data_type} ({type(data)}) to '{array_format}'.\"\n                \" Returning input data.\",\n            )\n        return data\n\n    @staticmethod\n    def _unpack_categories(series: pd.Series, categories: list) -&gt; pd.Series:\n        # nan-likes can not be explicitly specified as a category\n        def valid_category(cat: Any) -&gt; bool:\n            return isinstance(cat, str) or (cat is not None and not np.isnan(cat))\n\n        filtered_categories = [c for c in categories if valid_category(c)]\n        col = []\n        for x in series:\n            try:\n                col.append(categories[int(x)])\n            except (TypeError, ValueError):\n                col.append(np.nan)\n\n        # We require two lines to create a series of categories as detailed here:\n        # https://pandas.pydata.org/pandas-docs/version/0.24/user_guide/categorical.html#series-creation\n        raw_cat = pd.Categorical(col, ordered=True, categories=filtered_categories)\n        return pd.Series(raw_cat, index=series.index, name=series.name)\n\n    def get_data(  # noqa: C901, PLR0912, PLR0915\n        self,\n        target: list[str] | str | None = None,\n        include_row_id: bool = False,  # noqa: FBT001, FBT002\n        include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n        np.ndarray | pd.DataFrame | None,\n        list[bool],\n        list[str],\n    ]:\n        \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n        Parameters\n        ----------\n        target : string, List[str] or None (default=None)\n            Name of target column to separate from the data.\n            Splitting multiple columns is currently not supported.\n        include_row_id : boolean (default=False)\n            Whether to include row ids in the returned dataset.\n        include_ignore_attribute : boolean (default=False)\n            Whether to include columns that are marked as \"ignore\"\n            on the server in the dataset.\n        dataset_format : string (default='dataframe')\n            The format of returned dataset.\n            If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n            matrix. Support for ``array`` will be removed in 0.15.\n            If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n        Returns\n        -------\n        X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n            Dataset\n        y : ndarray or pd.Series, shape (n_samples, ) or None\n            Target column\n        categorical_indicator : boolean ndarray\n            Mask that indicate categorical features.\n        attribute_names : List[str]\n            List of attribute names.\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        data, categorical, attribute_names = self._load_data()\n\n        to_exclude = []\n        if not include_row_id and self.row_id_attribute is not None:\n            if isinstance(self.row_id_attribute, str):\n                to_exclude.append(self.row_id_attribute)\n            elif isinstance(self.row_id_attribute, Iterable):\n                to_exclude.extend(self.row_id_attribute)\n\n        if not include_ignore_attribute and self.ignore_attribute is not None:\n            if isinstance(self.ignore_attribute, str):\n                to_exclude.append(self.ignore_attribute)\n            elif isinstance(self.ignore_attribute, Iterable):\n                to_exclude.extend(self.ignore_attribute)\n\n        if len(to_exclude) &gt; 0:\n            logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n            keep = np.array([column not in to_exclude for column in attribute_names])\n            data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n            categorical = [cat for cat, k in zip(categorical, keep) if k]\n            attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n        if target is None:\n            data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n            targets = None\n        else:\n            if isinstance(target, str):\n                target = target.split(\",\") if \",\" in target else [target]\n            targets = np.array([column in target for column in attribute_names])\n            target_names = [column for column in attribute_names if column in target]\n            if np.sum(targets) &gt; 1:\n                raise NotImplementedError(\n                    \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n                )\n            target_categorical = [\n                cat for cat, column in zip(categorical, attribute_names) if column in target\n            ]\n            target_dtype = int if target_categorical[0] else float\n\n            if isinstance(data, pd.DataFrame):\n                x = data.iloc[:, ~targets]\n                y = data.iloc[:, targets]\n            else:\n                x = data[:, ~targets]\n                y = data[:, targets].astype(target_dtype)  # type: ignore\n\n            categorical = [cat for cat, t in zip(categorical, targets) if not t]\n            attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n            x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n            if dataset_format == \"array\" and scipy.sparse.issparse(y):\n                # scikit-learn requires dense representation of targets\n                y = np.asarray(y.todense()).astype(target_dtype)\n                # dense representation of single column sparse arrays become a 2-d array\n                # need to flatten it to a 1-d array for _convert_array_format()\n                y = y.squeeze()\n            y = self._convert_array_format(y, dataset_format, target_names)\n            y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n            if len(y.shape) &gt; 1 and y.shape[1] == 1:\n                # single column targets should be 1-d for both `array` and `dataframe` formats\n                assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n                y = y.squeeze()\n            data, targets = x, y\n\n        return data, targets, categorical, attribute_names  # type: ignore\n\n    def _load_features(self) -&gt; None:\n        \"\"\"Load the features metadata from the server and store it in the dataset object.\"\"\"\n        # Delayed Import to avoid circular imports or having to import all of dataset.functions to\n        # import OpenMLDataset.\n        from openml.datasets.functions import _get_dataset_features_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        features_file = _get_dataset_features_file(None, self.dataset_id)\n        self._features = _read_features(features_file)\n\n    def _load_qualities(self) -&gt; None:\n        \"\"\"Load qualities information from the server and store it in the dataset object.\"\"\"\n        # same reason as above for _load_features\n        from openml.datasets.functions import _get_dataset_qualities_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        qualities_file = _get_dataset_qualities_file(None, self.dataset_id)\n\n        if qualities_file is None:\n            self._no_qualities_found = True\n        else:\n            self._qualities = _read_qualities(qualities_file)\n\n    def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n        \"\"\"Reads the datasets arff to determine the class-labels.\n\n        If the task has no class labels (for example a regression problem)\n        it returns None. Necessary because the data returned by get_data\n        only contains the indices of the classes, while OpenML needs the real\n        classname when uploading the results of a run.\n\n        Parameters\n        ----------\n        target_name : str\n            Name of the target attribute\n\n        Returns\n        -------\n        list\n        \"\"\"\n        for feature in self.features.values():\n            if feature.name == target_name:\n                if feature.data_type == \"nominal\":\n                    return feature.nominal_values\n\n                if feature.data_type == \"string\":\n                    # Rel.: #1311\n                    # The target is invalid for a classification task if the feature type is string\n                    # and not nominal. For such miss-configured tasks, we silently fix it here as\n                    # we can safely interpreter string as nominal.\n                    df, *_ = self.get_data()\n                    return list(df[feature.name].unique())\n\n        return None\n\n    def get_features_by_type(  # noqa: C901\n        self,\n        data_type: str,\n        exclude: list[str] | None = None,\n        exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n        exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; list[int]:\n        \"\"\"\n        Return indices of features of a given type, e.g. all nominal features.\n        Optional parameters to exclude various features by index or ontology.\n\n        Parameters\n        ----------\n        data_type : str\n            The data type to return (e.g., nominal, numeric, date, string)\n        exclude : list(int)\n            List of columns to exclude from the return value\n        exclude_ignore_attribute : bool\n            Whether to exclude the defined ignore attributes (and adapt the\n            return values as if these indices are not present)\n        exclude_row_id_attribute : bool\n            Whether to exclude the defined row id attributes (and adapt the\n            return values as if these indices are not present)\n\n        Returns\n        -------\n        result : list\n            a list of indices that have the specified data type\n        \"\"\"\n        if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n            raise TypeError(\"Illegal feature type requested\")\n        if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n            raise TypeError(\"ignore_attribute should be a list\")\n        if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n            raise TypeError(\"row id attribute should be a str\")\n        if exclude is not None and not isinstance(exclude, list):\n            raise TypeError(\"Exclude should be a list\")\n            # assert all(isinstance(elem, str) for elem in exclude),\n            #            \"Exclude should be a list of strings\"\n        to_exclude = []\n        if exclude is not None:\n            to_exclude.extend(exclude)\n        if exclude_ignore_attribute and self.ignore_attribute is not None:\n            to_exclude.extend(self.ignore_attribute)\n        if exclude_row_id_attribute and self.row_id_attribute is not None:\n            to_exclude.append(self.row_id_attribute)\n\n        result = []\n        offset = 0\n        # this function assumes that everything in to_exclude will\n        # be 'excluded' from the dataset (hence the offset)\n        for idx in self.features:\n            name = self.features[idx].name\n            if name in to_exclude:\n                offset += 1\n            elif self.features[idx].data_type == data_type:\n                result.append(idx - offset)\n        return result\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Adds the 'dataset' to file elements.\"\"\"\n        file_elements: dict = {}\n        path = None if self.data_file is None else Path(self.data_file).absolute()\n\n        if self._dataset is not None:\n            file_elements[\"dataset\"] = self._dataset\n        elif path is not None and path.exists():\n            with path.open(\"rb\") as fp:\n                file_elements[\"dataset\"] = fp.read()\n\n            try:\n                dataset_utf8 = str(file_elements[\"dataset\"], encoding=\"utf8\")\n                arff.ArffDecoder().decode(dataset_utf8, encode_nominal=True)\n            except arff.ArffException as e:\n                raise ValueError(\"The file you have provided is not a valid arff file.\") from e\n\n        elif self.url is None:\n            raise ValueError(\"No valid url/path to the data file was given.\")\n        return file_elements\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.dataset_id = int(xml_response[\"oml:upload_data_set\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        props = [\n            \"id\",\n            \"name\",\n            \"version\",\n            \"description\",\n            \"format\",\n            \"creator\",\n            \"contributor\",\n            \"collection_date\",\n            \"upload_date\",\n            \"language\",\n            \"licence\",\n            \"url\",\n            \"default_target_attribute\",\n            \"row_id_attribute\",\n            \"ignore_attribute\",\n            \"version_label\",\n            \"citation\",\n            \"tag\",\n            \"visibility\",\n            \"original_data_url\",\n            \"paper_url\",\n            \"update_comment\",\n            \"md5_checksum\",\n        ]\n\n        prop_values = {}\n        for prop in props:\n            content = getattr(self, prop, None)\n            if content is not None:\n                prop_values[\"oml:\" + prop] = content\n\n        return {\n            \"oml:data_set_description\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **prop_values,\n            }\n        }\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset.features","title":"<code>features: dict[int, OpenMLDataFeature]</code>  <code>property</code>","text":"<p>Get the features of this dataset.</p>"},{"location":"reference/#openml.OpenMLDataset.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Get the dataset numeric id.</p>"},{"location":"reference/#openml.OpenMLDataset.qualities","title":"<code>qualities: dict[str, float] | None</code>  <code>property</code>","text":"<p>Get the qualities of this dataset.</p>"},{"location":"reference/#openml.OpenMLDataset.get_data","title":"<code>get_data(target=None, include_row_id=False, include_ignore_attribute=False, dataset_format='dataframe')</code>","text":"<p>Returns dataset content as dataframes or sparse matrices.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>(string, List[str] or None(default=None))</code> <p>Name of target column to separate from the data. Splitting multiple columns is currently not supported.</p> <code>None</code> <code>include_row_id</code> <code>boolean(default=False)</code> <p>Whether to include row ids in the returned dataset.</p> <code>False</code> <code>include_ignore_attribute</code> <code>boolean(default=False)</code> <p>Whether to include columns that are marked as \"ignore\" on the server in the dataset.</p> <code>False</code> <code>dataset_format</code> <code>string(default='dataframe')</code> <p>The format of returned dataset. If <code>array</code>, the returned dataset will be a NumPy array or a SciPy sparse matrix. Support for <code>array</code> will be removed in 0.15. If <code>dataframe</code>, the returned dataset will be a Pandas DataFrame.</p> <code>'dataframe'</code> <p>Returns:</p> Name Type Description <code>X</code> <code>ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)</code> <p>Dataset</p> <code>y</code> <code>(ndarray or Series, shape(n_samples) or None)</code> <p>Target column</p> <code>categorical_indicator</code> <code>boolean ndarray</code> <p>Mask that indicate categorical features.</p> <code>attribute_names</code> <code>List[str]</code> <p>List of attribute names.</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_data(  # noqa: C901, PLR0912, PLR0915\n    self,\n    target: list[str] | str | None = None,\n    include_row_id: bool = False,  # noqa: FBT001, FBT002\n    include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n    np.ndarray | pd.DataFrame | None,\n    list[bool],\n    list[str],\n]:\n    \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n    Parameters\n    ----------\n    target : string, List[str] or None (default=None)\n        Name of target column to separate from the data.\n        Splitting multiple columns is currently not supported.\n    include_row_id : boolean (default=False)\n        Whether to include row ids in the returned dataset.\n    include_ignore_attribute : boolean (default=False)\n        Whether to include columns that are marked as \"ignore\"\n        on the server in the dataset.\n    dataset_format : string (default='dataframe')\n        The format of returned dataset.\n        If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n        matrix. Support for ``array`` will be removed in 0.15.\n        If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n    Returns\n    -------\n    X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n        Dataset\n    y : ndarray or pd.Series, shape (n_samples, ) or None\n        Target column\n    categorical_indicator : boolean ndarray\n        Mask that indicate categorical features.\n    attribute_names : List[str]\n        List of attribute names.\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    data, categorical, attribute_names = self._load_data()\n\n    to_exclude = []\n    if not include_row_id and self.row_id_attribute is not None:\n        if isinstance(self.row_id_attribute, str):\n            to_exclude.append(self.row_id_attribute)\n        elif isinstance(self.row_id_attribute, Iterable):\n            to_exclude.extend(self.row_id_attribute)\n\n    if not include_ignore_attribute and self.ignore_attribute is not None:\n        if isinstance(self.ignore_attribute, str):\n            to_exclude.append(self.ignore_attribute)\n        elif isinstance(self.ignore_attribute, Iterable):\n            to_exclude.extend(self.ignore_attribute)\n\n    if len(to_exclude) &gt; 0:\n        logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n        keep = np.array([column not in to_exclude for column in attribute_names])\n        data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n        categorical = [cat for cat, k in zip(categorical, keep) if k]\n        attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n    if target is None:\n        data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n        targets = None\n    else:\n        if isinstance(target, str):\n            target = target.split(\",\") if \",\" in target else [target]\n        targets = np.array([column in target for column in attribute_names])\n        target_names = [column for column in attribute_names if column in target]\n        if np.sum(targets) &gt; 1:\n            raise NotImplementedError(\n                \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n            )\n        target_categorical = [\n            cat for cat, column in zip(categorical, attribute_names) if column in target\n        ]\n        target_dtype = int if target_categorical[0] else float\n\n        if isinstance(data, pd.DataFrame):\n            x = data.iloc[:, ~targets]\n            y = data.iloc[:, targets]\n        else:\n            x = data[:, ~targets]\n            y = data[:, targets].astype(target_dtype)  # type: ignore\n\n        categorical = [cat for cat, t in zip(categorical, targets) if not t]\n        attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n        x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n        if dataset_format == \"array\" and scipy.sparse.issparse(y):\n            # scikit-learn requires dense representation of targets\n            y = np.asarray(y.todense()).astype(target_dtype)\n            # dense representation of single column sparse arrays become a 2-d array\n            # need to flatten it to a 1-d array for _convert_array_format()\n            y = y.squeeze()\n        y = self._convert_array_format(y, dataset_format, target_names)\n        y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n        if len(y.shape) &gt; 1 and y.shape[1] == 1:\n            # single column targets should be 1-d for both `array` and `dataframe` formats\n            assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n            y = y.squeeze()\n        data, targets = x, y\n\n    return data, targets, categorical, attribute_names  # type: ignore\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset.get_features_by_type","title":"<code>get_features_by_type(data_type, exclude=None, exclude_ignore_attribute=True, exclude_row_id_attribute=True)</code>","text":"<p>Return indices of features of a given type, e.g. all nominal features. Optional parameters to exclude various features by index or ontology.</p> <p>Parameters:</p> Name Type Description Default <code>data_type</code> <code>str</code> <p>The data type to return (e.g., nominal, numeric, date, string)</p> required <code>exclude</code> <code>list(int)</code> <p>List of columns to exclude from the return value</p> <code>None</code> <code>exclude_ignore_attribute</code> <code>bool</code> <p>Whether to exclude the defined ignore attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <code>exclude_row_id_attribute</code> <code>bool</code> <p>Whether to exclude the defined row id attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <p>Returns:</p> Name Type Description <code>result</code> <code>list</code> <p>a list of indices that have the specified data type</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_features_by_type(  # noqa: C901\n    self,\n    data_type: str,\n    exclude: list[str] | None = None,\n    exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n    exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n) -&gt; list[int]:\n    \"\"\"\n    Return indices of features of a given type, e.g. all nominal features.\n    Optional parameters to exclude various features by index or ontology.\n\n    Parameters\n    ----------\n    data_type : str\n        The data type to return (e.g., nominal, numeric, date, string)\n    exclude : list(int)\n        List of columns to exclude from the return value\n    exclude_ignore_attribute : bool\n        Whether to exclude the defined ignore attributes (and adapt the\n        return values as if these indices are not present)\n    exclude_row_id_attribute : bool\n        Whether to exclude the defined row id attributes (and adapt the\n        return values as if these indices are not present)\n\n    Returns\n    -------\n    result : list\n        a list of indices that have the specified data type\n    \"\"\"\n    if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n        raise TypeError(\"Illegal feature type requested\")\n    if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n        raise TypeError(\"ignore_attribute should be a list\")\n    if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n        raise TypeError(\"row id attribute should be a str\")\n    if exclude is not None and not isinstance(exclude, list):\n        raise TypeError(\"Exclude should be a list\")\n        # assert all(isinstance(elem, str) for elem in exclude),\n        #            \"Exclude should be a list of strings\"\n    to_exclude = []\n    if exclude is not None:\n        to_exclude.extend(exclude)\n    if exclude_ignore_attribute and self.ignore_attribute is not None:\n        to_exclude.extend(self.ignore_attribute)\n    if exclude_row_id_attribute and self.row_id_attribute is not None:\n        to_exclude.append(self.row_id_attribute)\n\n    result = []\n    offset = 0\n    # this function assumes that everything in to_exclude will\n    # be 'excluded' from the dataset (hence the offset)\n    for idx in self.features:\n        name = self.features[idx].name\n        if name in to_exclude:\n            offset += 1\n        elif self.features[idx].data_type == data_type:\n            result.append(idx - offset)\n    return result\n</code></pre>"},{"location":"reference/#openml.OpenMLDataset.retrieve_class_labels","title":"<code>retrieve_class_labels(target_name='class')</code>","text":"<p>Reads the datasets arff to determine the class-labels.</p> <p>If the task has no class labels (for example a regression problem) it returns None. Necessary because the data returned by get_data only contains the indices of the classes, while OpenML needs the real classname when uploading the results of a run.</p> <p>Parameters:</p> Name Type Description Default <code>target_name</code> <code>str</code> <p>Name of the target attribute</p> <code>'class'</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n    \"\"\"Reads the datasets arff to determine the class-labels.\n\n    If the task has no class labels (for example a regression problem)\n    it returns None. Necessary because the data returned by get_data\n    only contains the indices of the classes, while OpenML needs the real\n    classname when uploading the results of a run.\n\n    Parameters\n    ----------\n    target_name : str\n        Name of the target attribute\n\n    Returns\n    -------\n    list\n    \"\"\"\n    for feature in self.features.values():\n        if feature.name == target_name:\n            if feature.data_type == \"nominal\":\n                return feature.nominal_values\n\n            if feature.data_type == \"string\":\n                # Rel.: #1311\n                # The target is invalid for a classification task if the feature type is string\n                # and not nominal. For such miss-configured tasks, we silently fix it here as\n                # we can safely interpreter string as nominal.\n                df, *_ = self.get_data()\n                return list(df[feature.name].unique())\n\n    return None\n</code></pre>"},{"location":"reference/#openml.OpenMLEvaluation","title":"<code>OpenMLEvaluation</code>","text":"<p>Contains all meta-information about a run / evaluation combination, according to the evaluation/list function</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Refers to the run.</p> required <code>task_id</code> <code>int</code> <p>Refers to the task.</p> required <code>setup_id</code> <code>int</code> <p>Refers to the setup.</p> required <code>flow_id</code> <code>int</code> <p>Refers to the flow.</p> required <code>flow_name</code> <code>str</code> <p>Name of the referred flow.</p> required <code>data_id</code> <code>int</code> <p>Refers to the dataset.</p> required <code>data_name</code> <code>str</code> <p>The name of the dataset.</p> required <code>function</code> <code>str</code> <p>The evaluation metric of this item (e.g., accuracy).</p> required <code>upload_time</code> <code>str</code> <p>The time of evaluation.</p> required <code>uploader</code> <code>int</code> <p>Uploader ID (user ID)</p> required <code>upload_name</code> <code>str</code> <p>Name of the uploader of this evaluation</p> required <code>value</code> <code>float</code> <p>The value (score) of this evaluation.</p> required <code>values</code> <code>List[float]</code> <p>The values (scores) per repeat and fold (if requested)</p> required <code>array_data</code> <code>str</code> <p>list of information per class. (e.g., in case of precision, auroc, recall)</p> <code>None</code> Source code in <code>openml/evaluations/evaluation.py</code> <pre><code>class OpenMLEvaluation:\n    \"\"\"\n    Contains all meta-information about a run / evaluation combination,\n    according to the evaluation/list function\n\n    Parameters\n    ----------\n    run_id : int\n        Refers to the run.\n    task_id : int\n        Refers to the task.\n    setup_id : int\n        Refers to the setup.\n    flow_id : int\n        Refers to the flow.\n    flow_name : str\n        Name of the referred flow.\n    data_id : int\n        Refers to the dataset.\n    data_name : str\n        The name of the dataset.\n    function : str\n        The evaluation metric of this item (e.g., accuracy).\n    upload_time : str\n        The time of evaluation.\n    uploader: int\n        Uploader ID (user ID)\n    upload_name : str\n        Name of the uploader of this evaluation\n    value : float\n        The value (score) of this evaluation.\n    values : List[float]\n        The values (scores) per repeat and fold (if requested)\n    array_data : str\n        list of information per class.\n        (e.g., in case of precision, auroc, recall)\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        run_id: int,\n        task_id: int,\n        setup_id: int,\n        flow_id: int,\n        flow_name: str,\n        data_id: int,\n        data_name: str,\n        function: str,\n        upload_time: str,\n        uploader: int,\n        uploader_name: str,\n        value: float | None,\n        values: list[float] | None,\n        array_data: str | None = None,\n    ):\n        self.run_id = run_id\n        self.task_id = task_id\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.data_id = data_id\n        self.data_name = data_name\n        self.function = function\n        self.upload_time = upload_time\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.value = value\n        self.values = values\n        self.array_data = array_data\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Evaluation\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Upload Date\": self.upload_time,\n            \"Run ID\": self.run_id,\n            \"OpenML Run URL\": openml.runs.OpenMLRun.url_for_id(self.run_id),\n            \"Task ID\": self.task_id,\n            \"OpenML Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"OpenML Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Setup ID\": self.setup_id,\n            \"Data ID\": self.data_id,\n            \"Data Name\": self.data_name,\n            \"OpenML Data URL\": openml.datasets.OpenMLDataset.url_for_id(self.data_id),\n            \"Metric Used\": self.function,\n            \"Result\": self.value,\n        }\n\n        order = [\n            \"Uploader Date\",\n            \"Run ID\",\n            \"OpenML Run URL\",\n            \"Task ID\",\n            \"OpenML Task URL\" \"Flow ID\",\n            \"OpenML Flow URL\",\n            \"Setup ID\",\n            \"Data ID\",\n            \"Data Name\",\n            \"OpenML Data URL\",\n            \"Metric Used\",\n            \"Result\",\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow","title":"<code>OpenMLFlow</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Flow. Stores machine learning models.</p> <p>Flows should not be generated manually, but by the function :meth:<code>openml.flows.create_flow_from_model</code>. Using this helper function ensures that all relevant fields are filled in.</p> <p>Implements <code>openml.implementation.upload.xsd &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/ openml.implementation.upload.xsd&gt;</code>_.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the flow. Is used together with the attribute <code>external_version</code> as a unique identifier of the flow.</p> required <code>description</code> <code>str</code> <p>Human-readable description of the flow (free text).</p> required <code>model</code> <code>object</code> <p>ML model which is described by this flow.</p> required <code>components</code> <code>OrderedDict</code> <p>Mapping from component identifier to an OpenMLFlow object. Components are usually subfunctions of an algorithm (e.g. kernels), base learners in ensemble algorithms (decision tree in adaboost) or building blocks of a machine learning pipeline. Components are modeled as independent flows and can be shared between flows (different pipelines can use the same components).</p> required <code>parameters</code> <code>OrderedDict</code> <p>Mapping from parameter name to the parameter default value. The parameter default value must be of type <code>str</code>, so that the respective toolbox plugin can take care of casting the parameter default value to the correct type.</p> required <code>parameters_meta_info</code> <code>OrderedDict</code> <p>Mapping from parameter name to <code>dict</code>. Stores additional information for each parameter. Required keys are <code>data_type</code> and <code>description</code>.</p> required <code>external_version</code> <code>str</code> <p>Version number of the software the flow is implemented in. Is used together with the attribute <code>name</code> as a uniquer identifier of the flow.</p> required <code>tags</code> <code>list</code> <p>List of tags. Created on the server by other API calls.</p> required <code>language</code> <code>str</code> <p>Natural language the flow is described in (not the programming language).</p> required <code>dependencies</code> <code>str</code> <p>A list of dependencies necessary to run the flow. This field should contain all libraries the flow depends on. To allow reproducibility it should also specify the exact version numbers.</p> required <code>class_name</code> <code>str</code> <p>The development language name of the class which is described by this flow.</p> <code>None</code> <code>custom_name</code> <code>str</code> <p>Custom name of the flow given by the owner.</p> <code>None</code> <code>binary_url</code> <code>str</code> <p>Url from which the binary can be downloaded. Added by the server. Ignored when uploaded manually. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_format</code> <code>str</code> <p>Format in which the binary code was uploaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_md5</code> <code>str</code> <p>MD5 checksum to check if the binary code was correctly downloaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>uploader</code> <code>str</code> <p>OpenML user ID of the uploader. Filled in by the server.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>Date the flow was uploaded. Filled in by the server.</p> <code>None</code> <code>flow_id</code> <code>int</code> <p>Flow ID. Assigned by the server.</p> <code>None</code> <code>extension</code> <code>Extension</code> <p>The extension for a flow (e.g., sklearn).</p> <code>None</code> <code>version</code> <code>str</code> <p>OpenML version of the flow. Assigned by the server.</p> <code>None</code> Source code in <code>openml/flows/flow.py</code> <pre><code>class OpenMLFlow(OpenMLBase):\n    \"\"\"OpenML Flow. Stores machine learning models.\n\n    Flows should not be generated manually, but by the function\n    :meth:`openml.flows.create_flow_from_model`. Using this helper function\n    ensures that all relevant fields are filled in.\n\n    Implements `openml.implementation.upload.xsd\n    &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/\n    openml.implementation.upload.xsd&gt;`_.\n\n    Parameters\n    ----------\n    name : str\n        Name of the flow. Is used together with the attribute\n        `external_version` as a unique identifier of the flow.\n    description : str\n        Human-readable description of the flow (free text).\n    model : object\n        ML model which is described by this flow.\n    components : OrderedDict\n        Mapping from component identifier to an OpenMLFlow object. Components\n        are usually subfunctions of an algorithm (e.g. kernels), base learners\n        in ensemble algorithms (decision tree in adaboost) or building blocks\n        of a machine learning pipeline. Components are modeled as independent\n        flows and can be shared between flows (different pipelines can use\n        the same components).\n    parameters : OrderedDict\n        Mapping from parameter name to the parameter default value. The\n        parameter default value must be of type `str`, so that the respective\n        toolbox plugin can take care of casting the parameter default value to\n        the correct type.\n    parameters_meta_info : OrderedDict\n        Mapping from parameter name to `dict`. Stores additional information\n        for each parameter. Required keys are `data_type` and `description`.\n    external_version : str\n        Version number of the software the flow is implemented in. Is used\n        together with the attribute `name` as a uniquer identifier of the flow.\n    tags : list\n        List of tags. Created on the server by other API calls.\n    language : str\n        Natural language the flow is described in (not the programming\n        language).\n    dependencies : str\n        A list of dependencies necessary to run the flow. This field should\n        contain all libraries the flow depends on. To allow reproducibility\n        it should also specify the exact version numbers.\n    class_name : str, optional\n        The development language name of the class which is described by this\n        flow.\n    custom_name : str, optional\n        Custom name of the flow given by the owner.\n    binary_url : str, optional\n        Url from which the binary can be downloaded. Added by the server.\n        Ignored when uploaded manually. Will not be used by the python API\n        because binaries aren't compatible across machines.\n    binary_format : str, optional\n        Format in which the binary code was uploaded. Will not be used by the\n        python API because binaries aren't compatible across machines.\n    binary_md5 : str, optional\n        MD5 checksum to check if the binary code was correctly downloaded. Will\n        not be used by the python API because binaries aren't compatible across\n        machines.\n    uploader : str, optional\n        OpenML user ID of the uploader. Filled in by the server.\n    upload_date : str, optional\n        Date the flow was uploaded. Filled in by the server.\n    flow_id : int, optional\n        Flow ID. Assigned by the server.\n    extension : Extension, optional\n        The extension for a flow (e.g., sklearn).\n    version : str, optional\n        OpenML version of the flow. Assigned by the server.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        name: str,\n        description: str,\n        model: object,\n        components: dict,\n        parameters: dict,\n        parameters_meta_info: dict,\n        external_version: str,\n        tags: list,\n        language: str,\n        dependencies: str,\n        class_name: str | None = None,\n        custom_name: str | None = None,\n        binary_url: str | None = None,\n        binary_format: str | None = None,\n        binary_md5: str | None = None,\n        uploader: str | None = None,\n        upload_date: str | None = None,\n        flow_id: int | None = None,\n        extension: Extension | None = None,\n        version: str | None = None,\n    ):\n        self.name = name\n        self.description = description\n        self.model = model\n\n        for variable, variable_name in [\n            [components, \"components\"],\n            [parameters, \"parameters\"],\n            [parameters_meta_info, \"parameters_meta_info\"],\n        ]:\n            if not isinstance(variable, (OrderedDict, dict)):\n                raise TypeError(\n                    f\"{variable_name} must be of type OrderedDict or dict, \"\n                    f\"but is {type(variable)}.\",\n                )\n\n        self.components = components\n        self.parameters = parameters\n        self.parameters_meta_info = parameters_meta_info\n        self.class_name = class_name\n\n        keys_parameters = set(parameters.keys())\n        keys_parameters_meta_info = set(parameters_meta_info.keys())\n        if len(keys_parameters.difference(keys_parameters_meta_info)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters, but not in \"\n                \"parameters_meta_info.\"\n                % str(keys_parameters.difference(keys_parameters_meta_info)),\n            )\n        if len(keys_parameters_meta_info.difference(keys_parameters)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters_meta_info, \"\n                \"but not in parameters.\"\n                % str(keys_parameters_meta_info.difference(keys_parameters)),\n            )\n\n        self.external_version = external_version\n        self.uploader = uploader\n\n        self.custom_name = custom_name\n        self.tags = tags if tags is not None else []\n        self.binary_url = binary_url\n        self.binary_format = binary_format\n        self.binary_md5 = binary_md5\n        self.version = version\n        self.upload_date = upload_date\n        self.language = language\n        self.dependencies = dependencies\n        self.flow_id = flow_id\n        if extension is None:\n            self._extension = get_extension_by_flow(self)\n        else:\n            self._extension = extension\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the flow.\"\"\"\n        return self.flow_id\n\n    @property\n    def extension(self) -&gt; Extension:\n        \"\"\"The extension of the flow (e.g., sklearn).\"\"\"\n        if self._extension is not None:\n            return self._extension\n\n        raise RuntimeError(\n            f\"No extension could be found for flow {self.flow_id}: {self.name}\",\n        )\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields = {\n            \"Flow Name\": self.name,\n            \"Flow Description\": self.description,\n            \"Dependencies\": self.dependencies,\n        }\n        if self.flow_id is not None:\n            fields[\"Flow URL\"] = self.openml_url if self.openml_url is not None else \"None\"\n            fields[\"Flow ID\"] = str(self.flow_id)\n            if self.version is not None:\n                fields[\"Flow ID\"] += f\" (version {self.version})\"\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.binary_url is not None:\n            fields[\"Binary URL\"] = self.binary_url\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Flow ID\",\n            \"Flow URL\",\n            \"Flow Name\",\n            \"Flow Description\",\n            \"Binary URL\",\n            \"Upload Date\",\n            \"Dependencies\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: C901, PLR0912\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        flow_container = OrderedDict()  # type: 'dict[str, dict]'\n        flow_dict = OrderedDict(\n            [(\"@xmlns:oml\", \"http://openml.org/openml\")],\n        )  # type: 'dict[str, list | str]'  # E501\n        flow_container[\"oml:flow\"] = flow_dict\n        _add_if_nonempty(flow_dict, \"oml:id\", self.flow_id)\n\n        for required in [\"name\", \"external_version\"]:\n            if getattr(self, required) is None:\n                raise ValueError(f\"self.{required} is required but None\")\n        for attribute in [\n            \"uploader\",\n            \"name\",\n            \"custom_name\",\n            \"class_name\",\n            \"version\",\n            \"external_version\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n        ]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        if not self.description:\n            logger = logging.getLogger(__name__)\n            logger.warning(\"Flow % has empty description\", self.name)\n\n        flow_parameters = []\n        for key in self.parameters:\n            param_dict = OrderedDict()  # type: 'OrderedDict[str, str]'\n            param_dict[\"oml:name\"] = key\n            meta_info = self.parameters_meta_info[key]\n\n            _add_if_nonempty(param_dict, \"oml:data_type\", meta_info[\"data_type\"])\n            param_dict[\"oml:default_value\"] = self.parameters[key]\n            _add_if_nonempty(param_dict, \"oml:description\", meta_info[\"description\"])\n\n            for key_, value in param_dict.items():\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n                if value is not None and not isinstance(value, str):\n                    raise ValueError(\n                        f\"Parameter value {value} cannot be serialized \"\n                        f\"because it is of type {type(value)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            flow_parameters.append(param_dict)\n\n        flow_dict[\"oml:parameter\"] = flow_parameters\n\n        components = []\n        for key in self.components:\n            component_dict = OrderedDict()  # type: 'OrderedDict[str, dict]'\n            component_dict[\"oml:identifier\"] = key\n            if self.components[key] in [\"passthrough\", \"drop\"]:\n                component_dict[\"oml:flow\"] = {\n                    \"oml-python:serialized_object\": \"component_reference\",\n                    \"value\": {\"key\": self.components[key], \"step_name\": self.components[key]},\n                }\n            else:\n                component_dict[\"oml:flow\"] = self.components[key]._to_dict()[\"oml:flow\"]\n\n            for key_ in component_dict:\n                # We only need to check if the key is a string, because the\n                # value is a flow. The flow itself is valid by recursion\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            components.append(component_dict)\n\n        flow_dict[\"oml:component\"] = components\n        flow_dict[\"oml:tag\"] = self.tags\n        for attribute in [\"binary_url\", \"binary_format\", \"binary_md5\"]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        return flow_container\n\n    @classmethod\n    def _from_dict(cls, xml_dict: dict) -&gt; OpenMLFlow:\n        \"\"\"Create a flow from an xml description.\n\n        Calls itself recursively to create :class:`OpenMLFlow` objects of\n        subflows (components).\n\n        XML definition of a flow is available at\n        https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd\n\n        Parameters\n        ----------\n        xml_dict : dict\n            Dictionary representation of the flow as created by _to_dict()\n\n        Returns\n        -------\n            OpenMLFlow\n\n        \"\"\"  # E501\n        arguments = OrderedDict()\n        dic = xml_dict[\"oml:flow\"]\n\n        # Mandatory parts in the xml file\n        for key in [\"name\"]:\n            arguments[key] = dic[\"oml:\" + key]\n\n        # non-mandatory parts in the xml file\n        for key in [\n            \"external_version\",\n            \"uploader\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n            \"version\",\n            \"binary_url\",\n            \"binary_format\",\n            \"binary_md5\",\n            \"class_name\",\n            \"custom_name\",\n        ]:\n            arguments[key] = dic.get(\"oml:\" + key)\n\n        # has to be converted to an int if present and cannot parsed in the\n        # two loops above\n        arguments[\"flow_id\"] = int(dic[\"oml:id\"]) if dic.get(\"oml:id\") is not None else None\n\n        # Now parse parts of a flow which can occur multiple times like\n        # parameters, components (subflows) and tags. These can't be tackled\n        # in the loops above because xmltodict returns a dict if such an\n        # entity occurs once, and a list if it occurs multiple times.\n        # Furthermore, they must be treated differently, for example\n        # for components this method is called recursively and\n        # for parameters the actual information is split into two dictionaries\n        # for easier access in python.\n\n        parameters = OrderedDict()\n        parameters_meta_info = OrderedDict()\n        if \"oml:parameter\" in dic:\n            # In case of a single parameter, xmltodict returns a dictionary,\n            # otherwise a list.\n            oml_parameters = extract_xml_tags(\"oml:parameter\", dic, allow_none=False)\n\n            for oml_parameter in oml_parameters:\n                parameter_name = oml_parameter[\"oml:name\"]\n                default_value = oml_parameter[\"oml:default_value\"]\n                parameters[parameter_name] = default_value\n\n                meta_info = OrderedDict()\n                meta_info[\"description\"] = oml_parameter.get(\"oml:description\")\n                meta_info[\"data_type\"] = oml_parameter.get(\"oml:data_type\")\n                parameters_meta_info[parameter_name] = meta_info\n        arguments[\"parameters\"] = parameters\n        arguments[\"parameters_meta_info\"] = parameters_meta_info\n\n        components = OrderedDict()\n        if \"oml:component\" in dic:\n            # In case of a single component xmltodict returns a dict,\n            # otherwise a list.\n            oml_components = extract_xml_tags(\"oml:component\", dic, allow_none=False)\n\n            for component in oml_components:\n                flow = OpenMLFlow._from_dict(component)\n                components[component[\"oml:identifier\"]] = flow\n        arguments[\"components\"] = components\n        arguments[\"tags\"] = extract_xml_tags(\"oml:tag\", dic)\n\n        arguments[\"model\"] = None\n        return cls(**arguments)\n\n    def to_filesystem(self, output_directory: str | Path) -&gt; None:\n        \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n        output_directory = Path(output_directory)\n        output_directory.mkdir(parents=True, exist_ok=True)\n\n        output_path = output_directory / \"flow.xml\"\n        if output_path.exists():\n            raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n        run_xml = self._to_xml()\n        with output_path.open(\"w\") as f:\n            f.write(run_xml)\n\n    @classmethod\n    def from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n        \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n        input_directory = Path(input_directory) / \"flow.xml\"\n        with input_directory.open() as f:\n            xml_string = f.read()\n        return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.flow_id = int(xml_response[\"oml:upload_flow\"][\"oml:id\"])\n\n    def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n        \"\"\"Publish this flow to OpenML server.\n\n        Raises a PyOpenMLError if the flow exists on the server, but\n        `self.flow_id` does not match the server known flow id.\n\n        Parameters\n        ----------\n        raise_error_if_exists : bool, optional (default=False)\n            If True, raise PyOpenMLError if the flow exists on the server.\n            If False, update the local flow to match the server flow.\n\n        Returns\n        -------\n        self : OpenMLFlow\n\n        \"\"\"\n        # Import at top not possible because of cyclic dependencies. In\n        # particular, flow.py tries to import functions.py in order to call\n        # get_flow(), while functions.py tries to import flow.py in order to\n        # instantiate an OpenMLFlow.\n        import openml.flows.functions\n\n        flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n        if not flow_id:\n            if self.flow_id:\n                raise openml.exceptions.PyOpenMLError(\n                    \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n                )\n            super().publish()\n            assert self.flow_id is not None  # for mypy\n            flow_id = self.flow_id\n        elif raise_error_if_exists:\n            error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n            raise openml.exceptions.PyOpenMLError(error_message)\n        elif self.flow_id is not None and self.flow_id != flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n            )\n\n        flow = openml.flows.functions.get_flow(flow_id)\n        _copy_server_fields(flow, self)\n        try:\n            openml.flows.functions.assert_flows_equal(\n                self,\n                flow,\n                flow.upload_date,\n                ignore_parameter_values=True,\n                ignore_custom_name_if_none=True,\n            )\n        except ValueError as e:\n            message = e.args[0]\n            raise ValueError(\n                \"The flow on the server is inconsistent with the local flow. \"\n                f\"The server flow ID is {flow_id}. Please check manually and remove \"\n                f\"the flow if necessary! Error is:\\n'{message}'\",\n            ) from e\n        return self\n\n    def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n        \"\"\"\n        Returns for each sub-component of the flow the path of identifiers\n        that should be traversed to reach this component. The resulting dict\n        maps a key (identifying a flow by either its id, name or fullname) to\n        the parameter prefix.\n\n        Parameters\n        ----------\n        key_item: str\n            The flow attribute that will be used to identify flows in the\n            structure. Allowed values {flow_id, name}\n\n        Returns\n        -------\n        dict[str, List[str]]\n            The flow structure\n        \"\"\"\n        if key_item not in [\"flow_id\", \"name\"]:\n            raise ValueError(\"key_item should be in {flow_id, name}\")\n        structure = {}\n        for key, sub_flow in self.components.items():\n            sub_structure = sub_flow.get_structure(key_item)\n            for flow_name, flow_sub_structure in sub_structure.items():\n                structure[flow_name] = [key, *flow_sub_structure]\n        structure[getattr(self, key_item)] = []\n        return structure\n\n    def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n        \"\"\"\n        Returns a subflow from the tree of dependencies.\n\n        Parameters\n        ----------\n        structure: list[str]\n            A list of strings, indicating the location of the subflow\n\n        Returns\n        -------\n        OpenMLFlow\n            The OpenMLFlow that corresponds to the structure\n        \"\"\"\n        # make a copy of structure, as we don't want to change it in the\n        # outer scope\n        structure = list(structure)\n        if len(structure) &lt; 1:\n            raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n        sub_identifier = structure[0]\n        if sub_identifier not in self.components:\n            raise ValueError(\n                f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n            )\n        if len(structure) == 1:\n            return self.components[sub_identifier]  # type: ignore\n\n        structure.pop(0)\n        return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.extension","title":"<code>extension: Extension</code>  <code>property</code>","text":"<p>The extension of the flow (e.g., sklearn).</p>"},{"location":"reference/#openml.OpenMLFlow.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the flow.</p>"},{"location":"reference/#openml.OpenMLFlow.from_filesystem","title":"<code>from_filesystem(input_directory)</code>  <code>classmethod</code>","text":"<p>Read a flow from an XML in input_directory on the filesystem.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n    \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n    input_directory = Path(input_directory) / \"flow.xml\"\n    with input_directory.open() as f:\n        xml_string = f.read()\n    return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.get_structure","title":"<code>get_structure(key_item)</code>","text":"<p>Returns for each sub-component of the flow the path of identifiers that should be traversed to reach this component. The resulting dict maps a key (identifying a flow by either its id, name or fullname) to the parameter prefix.</p> <p>Parameters:</p> Name Type Description Default <code>key_item</code> <code>str</code> <p>The flow attribute that will be used to identify flows in the structure. Allowed values {flow_id, name}</p> required <p>Returns:</p> Type Description <code>dict[str, List[str]]</code> <p>The flow structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n    \"\"\"\n    Returns for each sub-component of the flow the path of identifiers\n    that should be traversed to reach this component. The resulting dict\n    maps a key (identifying a flow by either its id, name or fullname) to\n    the parameter prefix.\n\n    Parameters\n    ----------\n    key_item: str\n        The flow attribute that will be used to identify flows in the\n        structure. Allowed values {flow_id, name}\n\n    Returns\n    -------\n    dict[str, List[str]]\n        The flow structure\n    \"\"\"\n    if key_item not in [\"flow_id\", \"name\"]:\n        raise ValueError(\"key_item should be in {flow_id, name}\")\n    structure = {}\n    for key, sub_flow in self.components.items():\n        sub_structure = sub_flow.get_structure(key_item)\n        for flow_name, flow_sub_structure in sub_structure.items():\n            structure[flow_name] = [key, *flow_sub_structure]\n    structure[getattr(self, key_item)] = []\n    return structure\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.get_subflow","title":"<code>get_subflow(structure)</code>","text":"<p>Returns a subflow from the tree of dependencies.</p> <p>Parameters:</p> Name Type Description Default <code>structure</code> <code>list[str]</code> <p>A list of strings, indicating the location of the subflow</p> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> <p>The OpenMLFlow that corresponds to the structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n    \"\"\"\n    Returns a subflow from the tree of dependencies.\n\n    Parameters\n    ----------\n    structure: list[str]\n        A list of strings, indicating the location of the subflow\n\n    Returns\n    -------\n    OpenMLFlow\n        The OpenMLFlow that corresponds to the structure\n    \"\"\"\n    # make a copy of structure, as we don't want to change it in the\n    # outer scope\n    structure = list(structure)\n    if len(structure) &lt; 1:\n        raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n    sub_identifier = structure[0]\n    if sub_identifier not in self.components:\n        raise ValueError(\n            f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n        )\n    if len(structure) == 1:\n        return self.components[sub_identifier]  # type: ignore\n\n    structure.pop(0)\n    return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.publish","title":"<code>publish(raise_error_if_exists=False)</code>","text":"<p>Publish this flow to OpenML server.</p> <p>Raises a PyOpenMLError if the flow exists on the server, but <code>self.flow_id</code> does not match the server known flow id.</p> <p>Parameters:</p> Name Type Description Default <code>raise_error_if_exists</code> <code>(bool, optional(default=False))</code> <p>If True, raise PyOpenMLError if the flow exists on the server. If False, update the local flow to match the server flow.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>self</code> <code>OpenMLFlow</code> Source code in <code>openml/flows/flow.py</code> <pre><code>def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Publish this flow to OpenML server.\n\n    Raises a PyOpenMLError if the flow exists on the server, but\n    `self.flow_id` does not match the server known flow id.\n\n    Parameters\n    ----------\n    raise_error_if_exists : bool, optional (default=False)\n        If True, raise PyOpenMLError if the flow exists on the server.\n        If False, update the local flow to match the server flow.\n\n    Returns\n    -------\n    self : OpenMLFlow\n\n    \"\"\"\n    # Import at top not possible because of cyclic dependencies. In\n    # particular, flow.py tries to import functions.py in order to call\n    # get_flow(), while functions.py tries to import flow.py in order to\n    # instantiate an OpenMLFlow.\n    import openml.flows.functions\n\n    flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n    if not flow_id:\n        if self.flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n            )\n        super().publish()\n        assert self.flow_id is not None  # for mypy\n        flow_id = self.flow_id\n    elif raise_error_if_exists:\n        error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n        raise openml.exceptions.PyOpenMLError(error_message)\n    elif self.flow_id is not None and self.flow_id != flow_id:\n        raise openml.exceptions.PyOpenMLError(\n            \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n        )\n\n    flow = openml.flows.functions.get_flow(flow_id)\n    _copy_server_fields(flow, self)\n    try:\n        openml.flows.functions.assert_flows_equal(\n            self,\n            flow,\n            flow.upload_date,\n            ignore_parameter_values=True,\n            ignore_custom_name_if_none=True,\n        )\n    except ValueError as e:\n        message = e.args[0]\n        raise ValueError(\n            \"The flow on the server is inconsistent with the local flow. \"\n            f\"The server flow ID is {flow_id}. Please check manually and remove \"\n            f\"the flow if necessary! Error is:\\n'{message}'\",\n        ) from e\n    return self\n</code></pre>"},{"location":"reference/#openml.OpenMLFlow.to_filesystem","title":"<code>to_filesystem(output_directory)</code>","text":"<p>Write a flow to the filesystem as XML to output_directory.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def to_filesystem(self, output_directory: str | Path) -&gt; None:\n    \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n    output_directory = Path(output_directory)\n    output_directory.mkdir(parents=True, exist_ok=True)\n\n    output_path = output_directory / \"flow.xml\"\n    if output_path.exists():\n        raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n    run_xml = self._to_xml()\n    with output_path.open(\"w\") as f:\n        f.write(run_xml)\n</code></pre>"},{"location":"reference/#openml.OpenMLLearningCurveTask","title":"<code>OpenMLLearningCurveTask</code>","text":"<p>               Bases: <code>OpenMLClassificationTask</code></p> <p>OpenML Learning Curve object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Learning Curve task.</p> required <code>task_type</code> <code>str</code> <p>Name of the Learning Curve task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the dataset that this task is associated with.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature in the dataset.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure to use for evaluating models.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Additional parameters for the estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the file containing the data splits for Learning Curve task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Learning Curve task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure to use for evaluating models.</p> <code>None</code> <code>class_labels</code> <code>list of str</code> <p>Class labels for Learning Curve tasks.</p> <code>None</code> <code>cost_matrix</code> <code>numpy array</code> <p>Cost matrix for Learning Curve tasks.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLLearningCurveTask(OpenMLClassificationTask):\n    \"\"\"OpenML Learning Curve object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Learning Curve task.\n    task_type : str\n        Name of the Learning Curve task.\n    data_set_id : int\n        ID of the dataset that this task is associated with.\n    target_name : str\n        Name of the target feature in the dataset.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure to use for evaluating models.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Additional parameters for the estimation procedure.\n    data_splits_url : str, default=None\n        URL of the file containing the data splits for Learning Curve task.\n    task_id : Union[int, None]\n        ID of the Learning Curve task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure to use for evaluating models.\n    class_labels : list of str, default=None\n        Class labels for Learning Curve tasks.\n    cost_matrix : numpy array, default=None\n        Cost matrix for Learning Curve tasks.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 13,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n            class_labels=class_labels,\n            cost_matrix=cost_matrix,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLParameter","title":"<code>OpenMLParameter</code>","text":"<p>Parameter object (used in setup).</p> <p>Parameters:</p> Name Type Description Default <code>input_id</code> <code>int</code> <p>The input id from the openml database</p> required <code>flow</code> <p>The flow to which this parameter is associated</p> required <code>flow</code> <p>The name of the flow (no version number) to which this parameter is associated</p> required <code>full_name</code> <code>str</code> <p>The name of the flow and parameter combined</p> required <code>parameter_name</code> <code>str</code> <p>The name of the parameter</p> required <code>data_type</code> <code>str</code> <p>The datatype of the parameter. generally unused for sklearn flows</p> required <code>default_value</code> <code>str</code> <p>The default value. For sklearn parameters, this is unknown and a default value is selected arbitrarily</p> required <code>value</code> <code>str</code> <p>If the parameter was set, the value that it was set to.</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLParameter:\n    \"\"\"Parameter object (used in setup).\n\n    Parameters\n    ----------\n    input_id : int\n        The input id from the openml database\n    flow id : int\n        The flow to which this parameter is associated\n    flow name : str\n        The name of the flow (no version number) to which this parameter\n        is associated\n    full_name : str\n        The name of the flow and parameter combined\n    parameter_name : str\n        The name of the parameter\n    data_type : str\n        The datatype of the parameter. generally unused for sklearn flows\n    default_value : str\n        The default value. For sklearn parameters, this is unknown and a\n        default value is selected arbitrarily\n    value : str\n        If the parameter was set, the value that it was set to.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        input_id: int,\n        flow_id: int,\n        flow_name: str,\n        full_name: str,\n        parameter_name: str,\n        data_type: str,\n        default_value: str,\n        value: str,\n    ):\n        self.id = input_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.full_name = full_name\n        self.parameter_name = parameter_name\n        self.data_type = data_type\n        self.default_value = default_value\n        self.value = value\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Parameter\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"ID\": self.id,\n            \"Flow ID\": self.flow_id,\n            # \"Flow Name\": self.flow_name,\n            \"Flow Name\": self.full_name,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Parameter Name\": self.parameter_name,\n        }\n        # indented prints for parameter attributes\n        # indention = 2 spaces + 1 | + 2 underscores\n        indent = \"{}|{}\".format(\" \" * 2, \"_\" * 2)\n        parameter_data_type = f\"{indent}Data Type\"\n        fields[parameter_data_type] = self.data_type\n        parameter_default = f\"{indent}Default\"\n        fields[parameter_default] = self.default_value\n        parameter_value = f\"{indent}Value\"\n        fields[parameter_value] = self.value\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Parameter Name\",\n            parameter_data_type,\n            parameter_default,\n            parameter_value,\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/#openml.OpenMLRegressionTask","title":"<code>OpenMLRegressionTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Regression object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML Regression task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML Regression task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature used in the Regression task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the Regression task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML Regression task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the Regression task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLRegressionTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Regression object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML Regression task.\n    task_type : str\n        Task type of the OpenML Regression task.\n    data_set_id : int\n        ID of the OpenML dataset.\n    target_name : str\n        Name of the target feature used in the Regression task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the Regression task.\n    task_id : Union[int, None]\n        ID of the OpenML Regression task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the Regression task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 7,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLRun","title":"<code>OpenMLRun</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Run: result of running a model on an OpenML dataset.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The ID of the OpenML task associated with the run.</p> required <code>flow_id</code> <code>int | None</code> <p>The ID of the OpenML flow associated with the run.</p> required <code>dataset_id</code> <code>int | None</code> <p>The ID of the OpenML dataset used for the run.</p> required <code>setup_string</code> <code>str | None</code> <p>The setup string of the run.</p> <code>None</code> <code>output_files</code> <code>dict[str, int] | None</code> <p>Specifies where each related file can be found.</p> <code>None</code> <code>setup_id</code> <code>int | None</code> <p>An integer representing the ID of the setup used for the run.</p> <code>None</code> <code>tags</code> <code>list[str] | None</code> <p>Representing the tags associated with the run.</p> <code>None</code> <code>uploader</code> <code>int | None</code> <p>User ID of the uploader.</p> <code>None</code> <code>uploader_name</code> <code>str | None</code> <p>The name of the person who uploaded the run.</p> <code>None</code> <code>evaluations</code> <code>dict | None</code> <p>Representing the evaluations of the run.</p> <code>None</code> <code>fold_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each fold.</p> <code>None</code> <code>sample_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each sample.</p> <code>None</code> <code>data_content</code> <code>list[list] | None</code> <p>The predictions generated from executing this run.</p> <code>None</code> <code>trace</code> <code>OpenMLRunTrace | None</code> <p>The trace containing information on internal model evaluations of this run.</p> <code>None</code> <code>model</code> <code>object | None</code> <p>The untrained model that was evaluated in the run.</p> <code>None</code> <code>task_type</code> <code>str | None</code> <p>The type of the OpenML task associated with the run.</p> <code>None</code> <code>task_evaluation_measure</code> <code>str | None</code> <p>The evaluation measure used for the task.</p> <code>None</code> <code>flow_name</code> <code>str | None</code> <p>The name of the OpenML flow associated with the run.</p> <code>None</code> <code>parameter_settings</code> <code>list[dict[str, Any]] | None</code> <p>Representing the parameter settings used for the run.</p> <code>None</code> <code>predictions_url</code> <code>str | None</code> <p>The URL of the predictions file.</p> <code>None</code> <code>task</code> <code>OpenMLTask | None</code> <p>An instance of the OpenMLTask class, representing the OpenML task associated with the run.</p> <code>None</code> <code>flow</code> <code>OpenMLFlow | None</code> <p>An instance of the OpenMLFlow class, representing the OpenML flow associated with the run.</p> <code>None</code> <code>run_id</code> <code>int | None</code> <p>The ID of the run.</p> <code>None</code> <code>description_text</code> <code>str | None</code> <p>Description text to add to the predictions file. If left None, is set to the time the arff file is generated.</p> <code>None</code> <code>run_details</code> <code>str | None</code> <p>Description of the run stored in the run meta-data.</p> <code>None</code> Source code in <code>openml/runs/run.py</code> <pre><code>class OpenMLRun(OpenMLBase):\n    \"\"\"OpenML Run: result of running a model on an OpenML dataset.\n\n    Parameters\n    ----------\n    task_id: int\n        The ID of the OpenML task associated with the run.\n    flow_id: int\n        The ID of the OpenML flow associated with the run.\n    dataset_id: int\n        The ID of the OpenML dataset used for the run.\n    setup_string: str\n        The setup string of the run.\n    output_files: Dict[str, int]\n        Specifies where each related file can be found.\n    setup_id: int\n        An integer representing the ID of the setup used for the run.\n    tags: List[str]\n        Representing the tags associated with the run.\n    uploader: int\n        User ID of the uploader.\n    uploader_name: str\n        The name of the person who uploaded the run.\n    evaluations: Dict\n        Representing the evaluations of the run.\n    fold_evaluations: Dict\n        The evaluations of the run for each fold.\n    sample_evaluations: Dict\n        The evaluations of the run for each sample.\n    data_content: List[List]\n        The predictions generated from executing this run.\n    trace: OpenMLRunTrace\n        The trace containing information on internal model evaluations of this run.\n    model: object\n        The untrained model that was evaluated in the run.\n    task_type: str\n        The type of the OpenML task associated with the run.\n    task_evaluation_measure: str\n        The evaluation measure used for the task.\n    flow_name: str\n        The name of the OpenML flow associated with the run.\n    parameter_settings: list[OrderedDict]\n        Representing the parameter settings used for the run.\n    predictions_url: str\n        The URL of the predictions file.\n    task: OpenMLTask\n        An instance of the OpenMLTask class, representing the OpenML task associated\n        with the run.\n    flow: OpenMLFlow\n        An instance of the OpenMLFlow class, representing the OpenML flow associated\n        with the run.\n    run_id: int\n        The ID of the run.\n    description_text: str, optional\n        Description text to add to the predictions file. If left None, is set to the\n        time the arff file is generated.\n    run_details: str, optional (default=None)\n        Description of the run stored in the run meta-data.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int,\n        flow_id: int | None,\n        dataset_id: int | None,\n        setup_string: str | None = None,\n        output_files: dict[str, int] | None = None,\n        setup_id: int | None = None,\n        tags: list[str] | None = None,\n        uploader: int | None = None,\n        uploader_name: str | None = None,\n        evaluations: dict | None = None,\n        fold_evaluations: dict | None = None,\n        sample_evaluations: dict | None = None,\n        data_content: list[list] | None = None,\n        trace: OpenMLRunTrace | None = None,\n        model: object | None = None,\n        task_type: str | None = None,\n        task_evaluation_measure: str | None = None,\n        flow_name: str | None = None,\n        parameter_settings: list[dict[str, Any]] | None = None,\n        predictions_url: str | None = None,\n        task: OpenMLTask | None = None,\n        flow: OpenMLFlow | None = None,\n        run_id: int | None = None,\n        description_text: str | None = None,\n        run_details: str | None = None,\n    ):\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.task_id = task_id\n        self.task_type = task_type\n        self.task_evaluation_measure = task_evaluation_measure\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.setup_id = setup_id\n        self.setup_string = setup_string\n        self.parameter_settings = parameter_settings\n        self.dataset_id = dataset_id\n        self.evaluations = evaluations\n        self.fold_evaluations = fold_evaluations\n        self.sample_evaluations = sample_evaluations\n        self.data_content = data_content\n        self.output_files = output_files\n        self.trace = trace\n        self.error_message = None\n        self.task = task\n        self.flow = flow\n        self.run_id = run_id\n        self.model = model\n        self.tags = tags\n        self.predictions_url = predictions_url\n        self.description_text = description_text\n        self.run_details = run_details\n        self._predictions = None\n\n    @property\n    def predictions(self) -&gt; pd.DataFrame:\n        \"\"\"Return a DataFrame with predictions for this run\"\"\"\n        if self._predictions is None:\n            if self.data_content:\n                arff_dict = self._generate_arff_dict()\n            elif self.predictions_url:\n                arff_text = openml._api_calls._download_text_file(self.predictions_url)\n                arff_dict = arff.loads(arff_text)\n            else:\n                raise RuntimeError(\"Run has no predictions.\")\n            self._predictions = pd.DataFrame(\n                arff_dict[\"data\"],\n                columns=[name for name, _ in arff_dict[\"attributes\"]],\n            )\n        return self._predictions\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the run, None if not uploaded to the server yet.\"\"\"\n        return self.run_id\n\n    def _evaluation_summary(self, metric: str) -&gt; str:\n        \"\"\"Summarizes the evaluation of a metric over all folds.\n\n        The fold scores for the metric must exist already. During run creation,\n        by default, the MAE for OpenMLRegressionTask and the accuracy for\n        OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed.\n\n        If repetition exist, we take the mean over all repetitions.\n\n        Parameters\n        ----------\n        metric: str\n            Name of an evaluation metric that was used to compute fold scores.\n\n        Returns\n        -------\n        metric_summary: str\n            A formatted string that displays the metric's evaluation summary.\n            The summary consists of the mean and std.\n        \"\"\"\n        if self.fold_evaluations is None:\n            raise ValueError(\"No fold evaluations available.\")\n        fold_score_lists = self.fold_evaluations[metric].values()\n\n        # Get the mean and std over all repetitions\n        rep_means = [np.mean(list(x.values())) for x in fold_score_lists]\n        rep_stds = [np.std(list(x.values())) for x in fold_score_lists]\n\n        return f\"{np.mean(rep_means):.4f} +- {np.mean(rep_stds):.4f}\"\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Set up fields\n        fields = {\n            \"Uploader Name\": self.uploader_name,\n            \"Metric\": self.task_evaluation_measure,\n            \"Run ID\": self.run_id,\n            \"Task ID\": self.task_id,\n            \"Task Type\": self.task_type,\n            \"Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"Flow Name\": self.flow_name,\n            \"Flow URL\": (\n                openml.flows.OpenMLFlow.url_for_id(self.flow_id)\n                if self.flow_id is not None\n                else None\n            ),\n            \"Setup ID\": self.setup_id,\n            \"Setup String\": self.setup_string,\n            \"Dataset ID\": self.dataset_id,\n            \"Dataset URL\": (\n                openml.datasets.OpenMLDataset.url_for_id(self.dataset_id)\n                if self.dataset_id is not None\n                else None\n            ),\n        }\n\n        # determines the order of the initial fields in which the information will be printed\n        order = [\"Uploader Name\", \"Uploader Profile\", \"Metric\", \"Result\"]\n\n        if self.uploader is not None:\n            fields[\"Uploader Profile\"] = f\"{openml.config.get_server_base_url()}/u/{self.uploader}\"\n        if self.run_id is not None:\n            fields[\"Run URL\"] = self.openml_url\n        if self.evaluations is not None and self.task_evaluation_measure in self.evaluations:\n            fields[\"Result\"] = self.evaluations[self.task_evaluation_measure]\n        elif self.fold_evaluations is not None:\n            # -- Add locally computed summary values if possible\n            if \"predictive_accuracy\" in self.fold_evaluations:\n                # OpenMLClassificationTask; OpenMLLearningCurveTask\n                result_field = \"Local Result - Accuracy (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"predictive_accuracy\")\n                order.append(result_field)\n            elif \"mean_absolute_error\" in self.fold_evaluations:\n                # OpenMLRegressionTask\n                result_field = \"Local Result - MAE (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"mean_absolute_error\")\n                order.append(result_field)\n\n            if \"usercpu_time_millis\" in self.fold_evaluations:\n                # Runtime should be available for most tasks types\n                rt_field = \"Local Runtime - ms (+- STD)\"\n                fields[rt_field] = self._evaluation_summary(\"usercpu_time_millis\")\n                order.append(rt_field)\n\n        # determines the remaining order\n        order += [\n            \"Run ID\",\n            \"Run URL\",\n            \"Task ID\",\n            \"Task Type\",\n            \"Task URL\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Setup ID\",\n            \"Setup String\",\n            \"Dataset ID\",\n            \"Dataset URL\",\n        ]\n        return [\n            (key, \"None\" if fields[key] is None else fields[key])  # type: ignore\n            for key in order\n            if key in fields\n        ]\n\n    @classmethod\n    def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n        \"\"\"\n        The inverse of the to_filesystem method. Instantiates an OpenMLRun\n        object based on files stored on the file system.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            are stored\n\n        expect_model : bool\n            if True, it requires the model pickle to be present, and an error\n            will be thrown if not. Otherwise, the model might or might not\n            be present.\n\n        Returns\n        -------\n        run : OpenMLRun\n            the re-instantiated run object\n        \"\"\"\n        # Avoiding cyclic imports\n        import openml.runs.functions\n\n        directory = Path(directory)\n        if not directory.is_dir():\n            raise ValueError(\"Could not find folder\")\n\n        description_path = directory / \"description.xml\"\n        predictions_path = directory / \"predictions.arff\"\n        trace_path = directory / \"trace.arff\"\n        model_path = directory / \"model.pkl\"\n\n        if not description_path.is_file():\n            raise ValueError(\"Could not find description.xml\")\n        if not predictions_path.is_file():\n            raise ValueError(\"Could not find predictions.arff\")\n        if (not model_path.is_file()) and expect_model:\n            raise ValueError(\"Could not find model.pkl\")\n\n        with description_path.open() as fht:\n            xml_string = fht.read()\n        run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n        if run.flow_id is None:\n            flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n            run.flow = flow\n            run.flow_name = flow.name\n\n        with predictions_path.open() as fht:\n            predictions = arff.load(fht)\n            run.data_content = predictions[\"data\"]\n\n        if model_path.is_file():\n            # note that it will load the model if the file exists, even if\n            # expect_model is False\n            with model_path.open(\"rb\") as fhb:\n                run.model = pickle.load(fhb)  # noqa: S301\n\n        if trace_path.is_file():\n            run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n        return run\n\n    def to_filesystem(\n        self,\n        directory: str | Path,\n        store_model: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        \"\"\"\n        The inverse of the from_filesystem method. Serializes a run\n        on the filesystem, to be uploaded later.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            will be stored. Should be empty\n\n        store_model : bool, optional (default=True)\n            if True, a model will be pickled as well. As this is the most\n            storage expensive part, it is often desirable to not store the\n            model.\n        \"\"\"\n        if self.data_content is None or self.model is None:\n            raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n        directory = Path(directory)\n        directory.mkdir(exist_ok=True, parents=True)\n\n        if any(directory.iterdir()):\n            raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n        run_xml = self._to_xml()\n        predictions_arff = arff.dumps(self._generate_arff_dict())\n\n        # It seems like typing does not allow to define the same variable multiple times\n        with (directory / \"description.xml\").open(\"w\") as fh:\n            fh.write(run_xml)\n        with (directory / \"predictions.arff\").open(\"w\") as fh:\n            fh.write(predictions_arff)\n        if store_model:\n            with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n                pickle.dump(self.model, fh_b)\n\n        if self.flow_id is None and self.flow is not None:\n            self.flow.to_filesystem(directory)\n\n        if self.trace is not None:\n            self.trace._to_filesystem(directory)\n\n    def _generate_arff_dict(self) -&gt; OrderedDict[str, Any]:\n        \"\"\"Generates the arff dictionary for uploading predictions to the\n        server.\n\n        Assumes that the run has been executed.\n\n        The order of the attributes follows the order defined by the Client API for R.\n\n        Returns\n        -------\n        arf_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains predictions and information about the run environment.\n        \"\"\"\n        if self.data_content is None:\n            raise ValueError(\"Run has not been executed.\")\n        if self.flow is None:\n            assert self.flow_id is not None, \"Run has no associated flow id!\"\n            self.flow = get_flow(self.flow_id)\n\n        if self.description_text is None:\n            self.description_text = time.strftime(\"%c\")\n        task = get_task(self.task_id)\n\n        arff_dict = OrderedDict()  # type: 'OrderedDict[str, Any]'\n        arff_dict[\"data\"] = self.data_content\n        arff_dict[\"description\"] = self.description_text\n        arff_dict[\"relation\"] = f\"openml_task_{task.task_id}_predictions\"\n\n        if isinstance(task, OpenMLLearningCurveTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"]\n                    + [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                    + [\n                        (\"confidence.\" + class_labels[i], \"NUMERIC\")\n                        for i in range(len(class_labels))\n                    ]\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLClassificationTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),  # Legacy\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                prediction_confidences = [\n                    (\"confidence.\" + class_labels[i], \"NUMERIC\") for i in range(len(class_labels))\n                ]\n                prediction_and_true = [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"] + prediction_and_true + prediction_confidences\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"prediction\", \"NUMERIC\"),\n                (\"truth\", \"NUMERIC\"),\n            ]\n\n        elif isinstance(task, OpenMLClusteringTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"cluster\", \"NUMERIC\"),\n            ]\n\n        else:\n            raise NotImplementedError(\"Task type %s is not yet supported.\" % str(task.task_type))\n\n        return arff_dict\n\n    def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n        \"\"\"Calculates metric scores based on predicted values. Assumes the\n        run has been executed locally (and contains run_data). Furthermore,\n        it assumes that the 'correct' or 'truth' attribute is specified in\n        the arff (which is an optional field, but always the case for\n        openml-python runs)\n\n        Parameters\n        ----------\n        sklearn_fn : function\n            a function pointer to a sklearn function that\n            accepts ``y_true``, ``y_pred`` and ``**kwargs``\n        kwargs : dict\n            kwargs for the function\n\n        Returns\n        -------\n        scores : ndarray of scores of length num_folds * num_repeats\n            metric results\n        \"\"\"\n        kwargs = kwargs if kwargs else {}\n        if self.data_content is not None and self.task_id is not None:\n            predictions_arff = self._generate_arff_dict()\n        elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n            predictions_file_url = openml._api_calls._file_id_to_url(\n                self.output_files[\"predictions\"],\n                \"predictions.arff\",\n            )\n            response = openml._api_calls._download_text_file(predictions_file_url)\n            predictions_arff = arff.loads(response)\n            # TODO: make this a stream reader\n        else:\n            raise ValueError(\n                \"Run should have been locally executed or \" \"contain outputfile reference.\",\n            )\n\n        # Need to know more about the task to compute scores correctly\n        task = get_task(self.task_id)\n\n        attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n        if (\n            task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n            and \"correct\" not in attribute_names\n        ):\n            raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n        if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n            raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n        if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n            raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n        def _attribute_list_to_dict(attribute_list):  # type: ignore\n            # convenience function: Creates a mapping to map from the name of\n            # attributes present in the arff prediction file to their index.\n            # This is necessary because the number of classes can be different\n            # for different tasks.\n            res = OrderedDict()\n            for idx in range(len(attribute_list)):\n                res[attribute_list[idx][0]] = idx\n            return res\n\n        attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n        repeat_idx = attribute_dict[\"repeat\"]\n        fold_idx = attribute_dict[\"fold\"]\n        predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n        if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n            correct_idx = attribute_dict[\"correct\"]\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            correct_idx = attribute_dict[\"truth\"]\n        has_samples = False\n        if \"sample\" in attribute_dict:\n            sample_idx = attribute_dict[\"sample\"]\n            has_samples = True\n\n        if (\n            predictions_arff[\"attributes\"][predicted_idx][1]\n            != predictions_arff[\"attributes\"][correct_idx][1]\n        ):\n            pred = predictions_arff[\"attributes\"][predicted_idx][1]\n            corr = predictions_arff[\"attributes\"][correct_idx][1]\n            raise ValueError(\n                \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n            )\n\n        # TODO: these could be cached\n        values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n        values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n        for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n            rep = line[repeat_idx]\n            fold = line[fold_idx]\n            samp = line[sample_idx] if has_samples else 0\n\n            if task.task_type_id in [\n                TaskType.SUPERVISED_CLASSIFICATION,\n                TaskType.LEARNING_CURVE,\n            ]:\n                prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                    line[predicted_idx],\n                )\n                correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n            elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n                prediction = line[predicted_idx]\n                correct = line[correct_idx]\n            if rep not in values_predict:\n                values_predict[rep] = OrderedDict()\n                values_correct[rep] = OrderedDict()\n            if fold not in values_predict[rep]:\n                values_predict[rep][fold] = OrderedDict()\n                values_correct[rep][fold] = OrderedDict()\n            if samp not in values_predict[rep][fold]:\n                values_predict[rep][fold][samp] = []\n                values_correct[rep][fold][samp] = []\n\n            values_predict[rep][fold][samp].append(prediction)\n            values_correct[rep][fold][samp].append(correct)\n\n        scores = []\n        for rep in values_predict:\n            for fold in values_predict[rep]:\n                last_sample = len(values_predict[rep][fold]) - 1\n                y_pred = values_predict[rep][fold][last_sample]\n                y_true = values_correct[rep][fold][last_sample]\n                scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n        return np.array(scores)\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.run_id = int(xml_response[\"oml:upload_run\"][\"oml:run_id\"])\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Get file_elements to upload to the server.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        if self.parameter_settings is None and self.model is None:\n            raise PyOpenMLError(\n                \"OpenMLRun must contain a model or be initialized with parameter_settings.\",\n            )\n        if self.flow_id is None:\n            if self.flow is None:\n                raise PyOpenMLError(\n                    \"OpenMLRun object does not contain a flow id or reference to OpenMLFlow \"\n                    \"(these should have been added while executing the task). \",\n                )\n\n            # publish the linked Flow before publishing the run.\n            self.flow.publish()\n            self.flow_id = self.flow.flow_id\n\n        if self.parameter_settings is None:\n            if self.flow is None:\n                assert self.flow_id is not None  # for mypy\n                self.flow = openml.flows.get_flow(self.flow_id)\n            self.parameter_settings = self.flow.extension.obtain_parameter_values(\n                self.flow,\n                self.model,\n            )\n\n        file_elements = {\"description\": (\"description.xml\", self._to_xml())}\n\n        if self.error_message is None:\n            predictions = arff.dumps(self._generate_arff_dict())\n            file_elements[\"predictions\"] = (\"predictions.arff\", predictions)\n\n        if self.trace is not None:\n            trace_arff = arff.dumps(self.trace.trace_to_arff())\n            file_elements[\"trace\"] = (\"trace.arff\", trace_arff)\n        return file_elements\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: PLR0912, C901\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        description = OrderedDict()  # type: 'OrderedDict'\n        description[\"oml:run\"] = OrderedDict()\n        description[\"oml:run\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n        description[\"oml:run\"][\"oml:task_id\"] = self.task_id\n        description[\"oml:run\"][\"oml:flow_id\"] = self.flow_id\n        if self.setup_string is not None:\n            description[\"oml:run\"][\"oml:setup_string\"] = self.setup_string\n        if self.error_message is not None:\n            description[\"oml:run\"][\"oml:error_message\"] = self.error_message\n        if self.run_details is not None:\n            description[\"oml:run\"][\"oml:run_details\"] = self.run_details\n        description[\"oml:run\"][\"oml:parameter_setting\"] = self.parameter_settings\n        if self.tags is not None:\n            description[\"oml:run\"][\"oml:tag\"] = self.tags\n        if (self.fold_evaluations is not None and len(self.fold_evaluations) &gt; 0) or (\n            self.sample_evaluations is not None and len(self.sample_evaluations) &gt; 0\n        ):\n            description[\"oml:run\"][\"oml:output_data\"] = OrderedDict()\n            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"] = []\n        if self.fold_evaluations is not None:\n            for measure in self.fold_evaluations:\n                for repeat in self.fold_evaluations[measure]:\n                    for fold, value in self.fold_evaluations[measure][repeat].items():\n                        current = OrderedDict(\n                            [\n                                (\"@repeat\", str(repeat)),\n                                (\"@fold\", str(fold)),\n                                (\"oml:name\", measure),\n                                (\"oml:value\", str(value)),\n                            ],\n                        )\n                        description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(current)\n        if self.sample_evaluations is not None:\n            for measure in self.sample_evaluations:\n                for repeat in self.sample_evaluations[measure]:\n                    for fold in self.sample_evaluations[measure][repeat]:\n                        for sample, value in self.sample_evaluations[measure][repeat][fold].items():\n                            current = OrderedDict(\n                                [\n                                    (\"@repeat\", str(repeat)),\n                                    (\"@fold\", str(fold)),\n                                    (\"@sample\", str(sample)),\n                                    (\"oml:name\", measure),\n                                    (\"oml:value\", str(value)),\n                                ],\n                            )\n                            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(\n                                current,\n                            )\n        return description\n</code></pre>"},{"location":"reference/#openml.OpenMLRun.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the run, None if not uploaded to the server yet.</p>"},{"location":"reference/#openml.OpenMLRun.predictions","title":"<code>predictions: pd.DataFrame</code>  <code>property</code>","text":"<p>Return a DataFrame with predictions for this run</p>"},{"location":"reference/#openml.OpenMLRun.from_filesystem","title":"<code>from_filesystem(directory, expect_model=True)</code>  <code>classmethod</code>","text":"<p>The inverse of the to_filesystem method. Instantiates an OpenMLRun object based on files stored on the file system.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results are stored</p> required <code>expect_model</code> <code>bool</code> <p>if True, it requires the model pickle to be present, and an error will be thrown if not. Otherwise, the model might or might not be present.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>the re-instantiated run object</p> Source code in <code>openml/runs/run.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n    \"\"\"\n    The inverse of the to_filesystem method. Instantiates an OpenMLRun\n    object based on files stored on the file system.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        are stored\n\n    expect_model : bool\n        if True, it requires the model pickle to be present, and an error\n        will be thrown if not. Otherwise, the model might or might not\n        be present.\n\n    Returns\n    -------\n    run : OpenMLRun\n        the re-instantiated run object\n    \"\"\"\n    # Avoiding cyclic imports\n    import openml.runs.functions\n\n    directory = Path(directory)\n    if not directory.is_dir():\n        raise ValueError(\"Could not find folder\")\n\n    description_path = directory / \"description.xml\"\n    predictions_path = directory / \"predictions.arff\"\n    trace_path = directory / \"trace.arff\"\n    model_path = directory / \"model.pkl\"\n\n    if not description_path.is_file():\n        raise ValueError(\"Could not find description.xml\")\n    if not predictions_path.is_file():\n        raise ValueError(\"Could not find predictions.arff\")\n    if (not model_path.is_file()) and expect_model:\n        raise ValueError(\"Could not find model.pkl\")\n\n    with description_path.open() as fht:\n        xml_string = fht.read()\n    run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n    if run.flow_id is None:\n        flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n        run.flow = flow\n        run.flow_name = flow.name\n\n    with predictions_path.open() as fht:\n        predictions = arff.load(fht)\n        run.data_content = predictions[\"data\"]\n\n    if model_path.is_file():\n        # note that it will load the model if the file exists, even if\n        # expect_model is False\n        with model_path.open(\"rb\") as fhb:\n            run.model = pickle.load(fhb)  # noqa: S301\n\n    if trace_path.is_file():\n        run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n    return run\n</code></pre>"},{"location":"reference/#openml.OpenMLRun.get_metric_fn","title":"<code>get_metric_fn(sklearn_fn, kwargs=None)</code>","text":"<p>Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' or 'truth' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs)</p> <p>Parameters:</p> Name Type Description Default <code>sklearn_fn</code> <code>function</code> <p>a function pointer to a sklearn function that accepts <code>y_true</code>, <code>y_pred</code> and <code>**kwargs</code></p> required <code>kwargs</code> <code>dict</code> <p>kwargs for the function</p> <code>None</code> <p>Returns:</p> Name Type Description <code>scores</code> <code>ndarray of scores of length num_folds * num_repeats</code> <p>metric results</p> Source code in <code>openml/runs/run.py</code> <pre><code>def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n    \"\"\"Calculates metric scores based on predicted values. Assumes the\n    run has been executed locally (and contains run_data). Furthermore,\n    it assumes that the 'correct' or 'truth' attribute is specified in\n    the arff (which is an optional field, but always the case for\n    openml-python runs)\n\n    Parameters\n    ----------\n    sklearn_fn : function\n        a function pointer to a sklearn function that\n        accepts ``y_true``, ``y_pred`` and ``**kwargs``\n    kwargs : dict\n        kwargs for the function\n\n    Returns\n    -------\n    scores : ndarray of scores of length num_folds * num_repeats\n        metric results\n    \"\"\"\n    kwargs = kwargs if kwargs else {}\n    if self.data_content is not None and self.task_id is not None:\n        predictions_arff = self._generate_arff_dict()\n    elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n        predictions_file_url = openml._api_calls._file_id_to_url(\n            self.output_files[\"predictions\"],\n            \"predictions.arff\",\n        )\n        response = openml._api_calls._download_text_file(predictions_file_url)\n        predictions_arff = arff.loads(response)\n        # TODO: make this a stream reader\n    else:\n        raise ValueError(\n            \"Run should have been locally executed or \" \"contain outputfile reference.\",\n        )\n\n    # Need to know more about the task to compute scores correctly\n    task = get_task(self.task_id)\n\n    attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n    if (\n        task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n        and \"correct\" not in attribute_names\n    ):\n        raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n    if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n        raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n    if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n        raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n    def _attribute_list_to_dict(attribute_list):  # type: ignore\n        # convenience function: Creates a mapping to map from the name of\n        # attributes present in the arff prediction file to their index.\n        # This is necessary because the number of classes can be different\n        # for different tasks.\n        res = OrderedDict()\n        for idx in range(len(attribute_list)):\n            res[attribute_list[idx][0]] = idx\n        return res\n\n    attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n    repeat_idx = attribute_dict[\"repeat\"]\n    fold_idx = attribute_dict[\"fold\"]\n    predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n    if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n        correct_idx = attribute_dict[\"correct\"]\n    elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n        correct_idx = attribute_dict[\"truth\"]\n    has_samples = False\n    if \"sample\" in attribute_dict:\n        sample_idx = attribute_dict[\"sample\"]\n        has_samples = True\n\n    if (\n        predictions_arff[\"attributes\"][predicted_idx][1]\n        != predictions_arff[\"attributes\"][correct_idx][1]\n    ):\n        pred = predictions_arff[\"attributes\"][predicted_idx][1]\n        corr = predictions_arff[\"attributes\"][correct_idx][1]\n        raise ValueError(\n            \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n        )\n\n    # TODO: these could be cached\n    values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n    values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n    for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n        rep = line[repeat_idx]\n        fold = line[fold_idx]\n        samp = line[sample_idx] if has_samples else 0\n\n        if task.task_type_id in [\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.LEARNING_CURVE,\n        ]:\n            prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                line[predicted_idx],\n            )\n            correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            prediction = line[predicted_idx]\n            correct = line[correct_idx]\n        if rep not in values_predict:\n            values_predict[rep] = OrderedDict()\n            values_correct[rep] = OrderedDict()\n        if fold not in values_predict[rep]:\n            values_predict[rep][fold] = OrderedDict()\n            values_correct[rep][fold] = OrderedDict()\n        if samp not in values_predict[rep][fold]:\n            values_predict[rep][fold][samp] = []\n            values_correct[rep][fold][samp] = []\n\n        values_predict[rep][fold][samp].append(prediction)\n        values_correct[rep][fold][samp].append(correct)\n\n    scores = []\n    for rep in values_predict:\n        for fold in values_predict[rep]:\n            last_sample = len(values_predict[rep][fold]) - 1\n            y_pred = values_predict[rep][fold][last_sample]\n            y_true = values_correct[rep][fold][last_sample]\n            scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n    return np.array(scores)\n</code></pre>"},{"location":"reference/#openml.OpenMLRun.to_filesystem","title":"<code>to_filesystem(directory, store_model=True)</code>","text":"<p>The inverse of the from_filesystem method. Serializes a run on the filesystem, to be uploaded later.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results will be stored. Should be empty</p> required <code>store_model</code> <code>(bool, optional(default=True))</code> <p>if True, a model will be pickled as well. As this is the most storage expensive part, it is often desirable to not store the model.</p> <code>True</code> Source code in <code>openml/runs/run.py</code> <pre><code>def to_filesystem(\n    self,\n    directory: str | Path,\n    store_model: bool = True,  # noqa: FBT001, FBT002\n) -&gt; None:\n    \"\"\"\n    The inverse of the from_filesystem method. Serializes a run\n    on the filesystem, to be uploaded later.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        will be stored. Should be empty\n\n    store_model : bool, optional (default=True)\n        if True, a model will be pickled as well. As this is the most\n        storage expensive part, it is often desirable to not store the\n        model.\n    \"\"\"\n    if self.data_content is None or self.model is None:\n        raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n    directory = Path(directory)\n    directory.mkdir(exist_ok=True, parents=True)\n\n    if any(directory.iterdir()):\n        raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n    run_xml = self._to_xml()\n    predictions_arff = arff.dumps(self._generate_arff_dict())\n\n    # It seems like typing does not allow to define the same variable multiple times\n    with (directory / \"description.xml\").open(\"w\") as fh:\n        fh.write(run_xml)\n    with (directory / \"predictions.arff\").open(\"w\") as fh:\n        fh.write(predictions_arff)\n    if store_model:\n        with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n            pickle.dump(self.model, fh_b)\n\n    if self.flow_id is None and self.flow is not None:\n        self.flow.to_filesystem(directory)\n\n    if self.trace is not None:\n        self.trace._to_filesystem(directory)\n</code></pre>"},{"location":"reference/#openml.OpenMLSetup","title":"<code>OpenMLSetup</code>","text":"<p>Setup object (a.k.a. Configuration).</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The OpenML setup id</p> required <code>flow_id</code> <code>int</code> <p>The flow that it is build upon</p> required <code>parameters</code> <code>dict</code> <p>The setting of the parameters</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLSetup:\n    \"\"\"Setup object (a.k.a. Configuration).\n\n    Parameters\n    ----------\n    setup_id : int\n        The OpenML setup id\n    flow_id : int\n        The flow that it is build upon\n    parameters : dict\n        The setting of the parameters\n    \"\"\"\n\n    def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None):\n        if not isinstance(setup_id, int):\n            raise ValueError(\"setup id should be int\")\n\n        if not isinstance(flow_id, int):\n            raise ValueError(\"flow id should be int\")\n\n        if parameters is not None and not isinstance(parameters, dict):\n            raise ValueError(\"parameters should be dict\")\n\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.parameters = parameters\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Setup\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Setup ID\": self.setup_id,\n            \"Flow ID\": self.flow_id,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"# of Parameters\": (\n                len(self.parameters) if self.parameters is not None else float(\"nan\")\n            ),\n        }\n\n        # determines the order in which the information will be printed\n        order = [\"Setup ID\", \"Flow ID\", \"Flow URL\", \"# of Parameters\"]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/#openml.OpenMLSplit","title":"<code>OpenMLSplit</code>","text":"<p>OpenML Split object.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>int or str</code> required <code>description</code> <code>str</code> required <code>split</code> <code>dict</code> required Source code in <code>openml/tasks/split.py</code> <pre><code>class OpenMLSplit:\n    \"\"\"OpenML Split object.\n\n    Parameters\n    ----------\n    name : int or str\n    description : str\n    split : dict\n    \"\"\"\n\n    def __init__(\n        self,\n        name: int | str,\n        description: str,\n        split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]],\n    ):\n        self.description = description\n        self.name = name\n        self.split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]] = {}\n\n        # Add splits according to repetition\n        for repetition in split:\n            _rep = int(repetition)\n            self.split[_rep] = OrderedDict()\n            for fold in split[_rep]:\n                self.split[_rep][fold] = OrderedDict()\n                for sample in split[_rep][fold]:\n                    self.split[_rep][fold][sample] = split[_rep][fold][sample]\n\n        self.repeats = len(self.split)\n\n        # TODO(eddiebergman): Better error message\n        if any(len(self.split[0]) != len(self.split[i]) for i in range(self.repeats)):\n            raise ValueError(\"\")\n\n        self.folds = len(self.split[0])\n        self.samples = len(self.split[0][0])\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if (\n            (not isinstance(self, type(other)))\n            or self.name != other.name\n            or self.description != other.description\n            or self.split.keys() != other.split.keys()\n            or any(\n                self.split[repetition].keys() != other.split[repetition].keys()\n                for repetition in self.split\n            )\n        ):\n            return False\n\n        samples = [\n            (repetition, fold, sample)\n            for repetition in self.split\n            for fold in self.split[repetition]\n            for sample in self.split[repetition][fold]\n        ]\n\n        for repetition, fold, sample in samples:\n            self_train, self_test = self.split[repetition][fold][sample]\n            other_train, other_test = other.split[repetition][fold][sample]\n            if not (np.all(self_train == other_train) and np.all(self_test == other_test)):\n                return False\n        return True\n\n    @classmethod\n    def _from_arff_file(cls, filename: Path) -&gt; OpenMLSplit:  # noqa: C901, PLR0912\n        repetitions = None\n        name = None\n\n        pkl_filename = filename.with_suffix(\".pkl.py3\")\n\n        if pkl_filename.exists():\n            with pkl_filename.open(\"rb\") as fh:\n                # TODO(eddiebergman): Would be good to figure out what _split is and assert it is\n                _split = pickle.load(fh)  # noqa: S301\n            repetitions = _split[\"repetitions\"]\n            name = _split[\"name\"]\n\n        # Cache miss\n        if repetitions is None:\n            # Faster than liac-arff and sufficient in this situation!\n            if not filename.exists():\n                raise FileNotFoundError(f\"Split arff {filename} does not exist!\")\n\n            file_data = arff.load(filename.open(\"r\"), return_type=arff.DENSE_GEN)\n            splits = file_data[\"data\"]\n            name = file_data[\"relation\"]\n            attrnames = [attr[0] for attr in file_data[\"attributes\"]]\n\n            repetitions = OrderedDict()\n\n            type_idx = attrnames.index(\"type\")\n            rowid_idx = attrnames.index(\"rowid\")\n            repeat_idx = attrnames.index(\"repeat\")\n            fold_idx = attrnames.index(\"fold\")\n            sample_idx = attrnames.index(\"sample\") if \"sample\" in attrnames else None\n\n            for line in splits:\n                # A line looks like type, rowid, repeat, fold\n                repetition = int(line[repeat_idx])\n                fold = int(line[fold_idx])\n                sample = 0\n                if sample_idx is not None:\n                    sample = int(line[sample_idx])\n\n                if repetition not in repetitions:\n                    repetitions[repetition] = OrderedDict()\n                if fold not in repetitions[repetition]:\n                    repetitions[repetition][fold] = OrderedDict()\n                if sample not in repetitions[repetition][fold]:\n                    repetitions[repetition][fold][sample] = ([], [])\n                split = repetitions[repetition][fold][sample]\n\n                type_ = line[type_idx]\n                if type_ == \"TRAIN\":\n                    split[0].append(line[rowid_idx])\n                elif type_ == \"TEST\":\n                    split[1].append(line[rowid_idx])\n                else:\n                    raise ValueError(type_)\n\n            for repetition in repetitions:\n                for fold in repetitions[repetition]:\n                    for sample in repetitions[repetition][fold]:\n                        repetitions[repetition][fold][sample] = Split(\n                            np.array(repetitions[repetition][fold][sample][0], dtype=np.int32),\n                            np.array(repetitions[repetition][fold][sample][1], dtype=np.int32),\n                        )\n\n            with pkl_filename.open(\"wb\") as fh:\n                pickle.dump({\"name\": name, \"repetitions\": repetitions}, fh, protocol=2)\n\n        assert name is not None\n        return cls(name, \"\", repetitions)\n\n    def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n        Parameters\n        ----------\n        repeat : int\n            Index of the repeat to retrieve.\n        fold : int\n            Index of the fold to retrieve.\n        sample : int\n            Index of the sample to retrieve.\n\n        Returns\n        -------\n        numpy.ndarray\n            The data split for the specified repeat, fold, and sample.\n\n        Raises\n        ------\n        ValueError\n            If the specified repeat, fold, or sample is not known.\n        \"\"\"\n        if repeat not in self.split:\n            raise ValueError(\"Repeat %s not known\" % str(repeat))\n        if fold not in self.split[repeat]:\n            raise ValueError(\"Fold %s not known\" % str(fold))\n        if sample not in self.split[repeat][fold]:\n            raise ValueError(\"Sample %s not known\" % str(sample))\n        return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/#openml.OpenMLSplit.get","title":"<code>get(repeat=0, fold=0, sample=0)</code>","text":"<p>Returns the specified data split from the CrossValidationSplit object.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>Index of the repeat to retrieve.</p> <code>0</code> <code>fold</code> <code>int</code> <p>Index of the fold to retrieve.</p> <code>0</code> <code>sample</code> <code>int</code> <p>Index of the sample to retrieve.</p> <code>0</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>The data split for the specified repeat, fold, and sample.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the specified repeat, fold, or sample is not known.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n    Parameters\n    ----------\n    repeat : int\n        Index of the repeat to retrieve.\n    fold : int\n        Index of the fold to retrieve.\n    sample : int\n        Index of the sample to retrieve.\n\n    Returns\n    -------\n    numpy.ndarray\n        The data split for the specified repeat, fold, and sample.\n\n    Raises\n    ------\n    ValueError\n        If the specified repeat, fold, or sample is not known.\n    \"\"\"\n    if repeat not in self.split:\n        raise ValueError(\"Repeat %s not known\" % str(repeat))\n    if fold not in self.split[repeat]:\n        raise ValueError(\"Fold %s not known\" % str(fold))\n    if sample not in self.split[repeat][fold]:\n        raise ValueError(\"Sample %s not known\" % str(sample))\n    return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/#openml.OpenMLStudy","title":"<code>OpenMLStudy</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLStudy represents the OpenML concept of a study (a collection of runs).</p> <p>It contains the following information: name, id, description, creation date, creator id and a list of run ids.</p> <p>According to this list of run ids, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLStudy(BaseStudy):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study (a collection of runs).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and a list of run ids.\n\n    According to this list of run ids, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        super().__init__(\n            study_id=study_id,\n            alias=alias,\n            main_entity_type=\"run\",\n            benchmark_suite=benchmark_suite,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=flows,\n            runs=runs,\n            setups=setups,\n        )\n</code></pre>"},{"location":"reference/#openml.OpenMLSupervisedTask","title":"<code>OpenMLSupervisedTask</code>","text":"<p>               Bases: <code>OpenMLTask</code>, <code>ABC</code></p> <p>OpenML Supervised Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature (the class variable).</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure for the task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure for the task.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the task.</p> <code>None</code> <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLSupervisedTask(OpenMLTask, ABC):\n    \"\"\"OpenML Supervised Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the task type.\n    task_type : str\n        Name of the task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the task.\n    target_name : str\n        Name of the target feature (the class variable).\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure for the task.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure for the task.\n    data_splits_url : str, default=None\n        URL of the data splits for the task.\n    task_id: Union[int, None]\n        Refers to the unique identifier of task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | scipy.sparse.spmatrix,\n        np.ndarray | None,\n    ]:\n        ...\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\"]\n    ) -&gt; tuple[\n        pd.DataFrame,\n        pd.Series | pd.DataFrame | None,\n    ]:\n        ...\n\n    # TODO(eddiebergman): Do all OpenMLSupervisedTask have a `y`?\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n        np.ndarray | pd.Series | pd.DataFrame | None,\n    ]:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        dataset = self.get_dataset()\n        if self.task_type_id not in (\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.SUPERVISED_REGRESSION,\n            TaskType.LEARNING_CURVE,\n        ):\n            raise NotImplementedError(self.task_type)\n\n        X, y, _, _ = dataset.get_data(\n            dataset_format=dataset_format,\n            target=self.target_name,\n        )\n        return X, y\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        task_container = super()._to_dict()\n        oml_input = task_container[\"oml:task_inputs\"][\"oml:input\"]  # type: ignore\n        assert isinstance(oml_input, list)\n\n        oml_input.append({\"@name\": \"target_feature\", \"#text\": self.target_name})\n        return task_container\n\n    @property\n    def estimation_parameters(self) -&gt; dict[str, str] | None:\n        \"\"\"Return the estimation parameters for the task.\"\"\"\n        warnings.warn(\n            \"The estimation_parameters attribute will be \"\n            \"deprecated in the future, please use \"\n            \"estimation_procedure['parameters'] instead\",\n            PendingDeprecationWarning,\n            stacklevel=2,\n        )\n        return self.estimation_procedure[\"parameters\"]\n\n    @estimation_parameters.setter\n    def estimation_parameters(self, est_parameters: dict[str, str] | None) -&gt; None:\n        self.estimation_procedure[\"parameters\"] = est_parameters\n</code></pre>"},{"location":"reference/#openml.OpenMLSupervisedTask.estimation_parameters","title":"<code>estimation_parameters: dict[str, str] | None</code>  <code>property</code> <code>writable</code>","text":"<p>Return the estimation parameters for the task.</p>"},{"location":"reference/#openml.OpenMLSupervisedTask.get_X_and_y","title":"<code>get_X_and_y(dataset_format='array')</code>","text":"<pre><code>get_X_and_y(dataset_format: Literal['array'] = 'array') -&gt; tuple[np.ndarray | scipy.sparse.spmatrix, np.ndarray | None]\n</code></pre><pre><code>get_X_and_y(dataset_format: Literal['dataframe']) -&gt; tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X_and_y(\n    self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n    np.ndarray | pd.Series | pd.DataFrame | None,\n]:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    dataset = self.get_dataset()\n    if self.task_type_id not in (\n        TaskType.SUPERVISED_CLASSIFICATION,\n        TaskType.SUPERVISED_REGRESSION,\n        TaskType.LEARNING_CURVE,\n    ):\n        raise NotImplementedError(self.task_type)\n\n    X, y, _, _ = dataset.get_data(\n        dataset_format=dataset_format,\n        target=self.target_name,\n    )\n    return X, y\n</code></pre>"},{"location":"reference/#openml.OpenMLTask","title":"<code>OpenMLTask</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Task object.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of OpenML task.</p> required <code>task_type_id</code> <code>TaskType</code> <p>Refers to the type of OpenML task.</p> required <code>task_type</code> <code>str</code> <p>Refers to the OpenML task.</p> required <code>data_set_id</code> <code>int</code> <p>Refers to the data.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>Refers to the type of estimates used.</p> <code>1</code> <code>estimation_procedure_type</code> <code>str | None</code> <p>Refers to the type of estimation procedure used for the OpenML task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict[str, str] | None</code> <p>Estimation parameters used for the OpenML task.</p> <code>None</code> <code>evaluation_measure</code> <code>str | None</code> <p>Refers to the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str | None</code> <p>Refers to the URL of the data splits used for the OpenML task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLTask(OpenMLBase):\n    \"\"\"OpenML Task object.\n\n    Parameters\n    ----------\n    task_id: Union[int, None]\n        Refers to the unique identifier of OpenML task.\n    task_type_id: TaskType\n        Refers to the type of OpenML task.\n    task_type: str\n        Refers to the OpenML task.\n    data_set_id: int\n        Refers to the data.\n    estimation_procedure_id: int\n        Refers to the type of estimates used.\n    estimation_procedure_type: str, default=None\n        Refers to the type of estimation procedure used for the OpenML task.\n    estimation_parameters: [Dict[str, str]], default=None\n        Estimation parameters used for the OpenML task.\n    evaluation_measure: str, default=None\n        Refers to the evaluation measure.\n    data_splits_url: str, default=None\n        Refers to the URL of the data splits used for the OpenML task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int | None,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n    ):\n        self.task_id = int(task_id) if task_id is not None else None\n        self.task_type_id = task_type_id\n        self.task_type = task_type\n        self.dataset_id = int(data_set_id)\n        self.evaluation_measure = evaluation_measure\n        self.estimation_procedure: _EstimationProcedure = {\n            \"type\": estimation_procedure_type,\n            \"parameters\": estimation_parameters,\n            \"data_splits_url\": data_splits_url,\n        }\n        self.estimation_procedure_id = estimation_procedure_id\n        self.split: OpenMLSplit | None = None\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"t\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the OpenML ID of this task.\"\"\"\n        return self.task_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        base_server_url = openml.config.get_server_base_url()\n        fields: dict[str, Any] = {\n            \"Task Type Description\": f\"{base_server_url}/tt/{self.task_type_id}\"\n        }\n        if self.task_id is not None:\n            fields[\"Task ID\"] = self.task_id\n            fields[\"Task URL\"] = self.openml_url\n        if self.evaluation_measure is not None:\n            fields[\"Evaluation Measure\"] = self.evaluation_measure\n        if self.estimation_procedure is not None:\n            fields[\"Estimation Procedure\"] = self.estimation_procedure[\"type\"]\n\n        # TODO(eddiebergman): Subclasses could advertise/provide this, instead of having to\n        # have the base class know about it's subclasses.\n        target_name = getattr(self, \"target_name\", None)\n        if target_name is not None:\n            fields[\"Target Feature\"] = target_name\n\n            class_labels = getattr(self, \"class_labels\", None)\n            if class_labels is not None:\n                fields[\"# of Classes\"] = len(class_labels)\n\n            if hasattr(self, \"cost_matrix\"):\n                fields[\"Cost Matrix\"] = \"Available\"\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Task Type Description\",\n            \"Task ID\",\n            \"Task URL\",\n            \"Estimation Procedure\",\n            \"Evaluation Measure\",\n            \"Target Feature\",\n            \"# of Classes\",\n            \"Cost Matrix\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def get_dataset(self) -&gt; datasets.OpenMLDataset:\n        \"\"\"Download dataset associated with task.\"\"\"\n        return datasets.get_dataset(self.dataset_id)\n\n    def get_train_test_split_indices(\n        self,\n        fold: int = 0,\n        repeat: int = 0,\n        sample: int = 0,\n    ) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n        # Replace with retrieve from cache\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.get(repeat=repeat, fold=fold, sample=sample)\n\n    def _download_split(self, cache_file: Path) -&gt; None:\n        # TODO(eddiebergman): Not sure about this try to read and error approach\n        try:\n            with cache_file.open(encoding=\"utf8\"):\n                pass\n        except OSError:\n            split_url = self.estimation_procedure[\"data_splits_url\"]\n            openml._api_calls._download_text_file(\n                source=str(split_url),\n                output_path=str(cache_file),\n            )\n\n    def download_split(self) -&gt; OpenMLSplit:\n        \"\"\"Download the OpenML split for a given task.\"\"\"\n        # TODO(eddiebergman): Can this every be `None`?\n        assert self.task_id is not None\n        cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n        cached_split_file = cache_dir / \"datasplits.arff\"\n\n        try:\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n        except OSError:\n            # Next, download and cache the associated split file\n            self._download_split(cached_split_file)\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n\n        return split\n\n    def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n        \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.repeats, self.split.folds, self.split.samples\n\n    # TODO(eddiebergman): Really need some better typing on all this\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        \"\"\"Creates a dictionary representation of self in a string format (for XML parsing).\"\"\"\n        oml_input = [\n            {\"@name\": \"source_data\", \"#text\": str(self.dataset_id)},\n            {\"@name\": \"estimation_procedure\", \"#text\": str(self.estimation_procedure_id)},\n        ]\n        if self.evaluation_measure is not None:  #\n            oml_input.append({\"@name\": \"evaluation_measures\", \"#text\": self.evaluation_measure})\n\n        return {\n            \"oml:task_inputs\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                \"oml:task_type_id\": self.task_type_id.value,  # This is an int from the enum?\n                \"oml:input\": oml_input,\n            }\n        }\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.task_id = int(xml_response[\"oml:upload_task\"][\"oml:id\"])\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the OpenML ID of this task.</p>"},{"location":"reference/#openml.OpenMLTask.download_split","title":"<code>download_split()</code>","text":"<p>Download the OpenML split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def download_split(self) -&gt; OpenMLSplit:\n    \"\"\"Download the OpenML split for a given task.\"\"\"\n    # TODO(eddiebergman): Can this every be `None`?\n    assert self.task_id is not None\n    cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n    cached_split_file = cache_dir / \"datasplits.arff\"\n\n    try:\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n    except OSError:\n        # Next, download and cache the associated split file\n        self._download_split(cached_split_file)\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n\n    return split\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.get_dataset","title":"<code>get_dataset()</code>","text":"<p>Download dataset associated with task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_dataset(self) -&gt; datasets.OpenMLDataset:\n    \"\"\"Download dataset associated with task.\"\"\"\n    return datasets.get_dataset(self.dataset_id)\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.get_split_dimensions","title":"<code>get_split_dimensions()</code>","text":"<p>Get the (repeats, folds, samples) of the split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n    \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.repeats, self.split.folds, self.split.samples\n</code></pre>"},{"location":"reference/#openml.OpenMLTask.get_train_test_split_indices","title":"<code>get_train_test_split_indices(fold=0, repeat=0, sample=0)</code>","text":"<p>Get the indices of the train and test splits for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_train_test_split_indices(\n    self,\n    fold: int = 0,\n    repeat: int = 0,\n    sample: int = 0,\n) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n    # Replace with retrieve from cache\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.get(repeat=repeat, fold=fold, sample=sample)\n</code></pre>"},{"location":"reference/#openml.populate_cache","title":"<code>populate_cache(task_ids=None, dataset_ids=None, flow_ids=None, run_ids=None)</code>","text":"<p>Populate a cache for offline and parallel usage of the OpenML connector.</p> <p>Parameters:</p> Name Type Description Default <code>task_ids</code> <code>iterable</code> <code>None</code> <code>dataset_ids</code> <code>iterable</code> <code>None</code> <code>flow_ids</code> <code>iterable</code> <code>None</code> <code>run_ids</code> <code>iterable</code> <code>None</code> <p>Returns:</p> Type Description <code>None</code> Source code in <code>openml/__init__.py</code> <pre><code>def populate_cache(\n    task_ids: list[int] | None = None,\n    dataset_ids: list[int | str] | None = None,\n    flow_ids: list[int] | None = None,\n    run_ids: list[int] | None = None,\n) -&gt; None:\n    \"\"\"\n    Populate a cache for offline and parallel usage of the OpenML connector.\n\n    Parameters\n    ----------\n    task_ids : iterable\n\n    dataset_ids : iterable\n\n    flow_ids : iterable\n\n    run_ids : iterable\n\n    Returns\n    -------\n    None\n    \"\"\"\n    if task_ids is not None:\n        for task_id in task_ids:\n            tasks.functions.get_task(task_id)\n\n    if dataset_ids is not None:\n        for dataset_id in dataset_ids:\n            datasets.functions.get_dataset(dataset_id)\n\n    if flow_ids is not None:\n        for flow_id in flow_ids:\n            flows.functions.get_flow(flow_id)\n\n    if run_ids is not None:\n        for run_id in run_ids:\n            runs.functions.get_run(run_id)\n</code></pre>"},{"location":"reference/SUMMARY/","title":"SUMMARY","text":"<ul> <li>base</li> <li>cli</li> <li>config</li> <li>datasets<ul> <li>data_feature</li> <li>dataset</li> <li>functions</li> </ul> </li> <li>evaluations<ul> <li>evaluation</li> <li>functions</li> </ul> </li> <li>exceptions</li> <li>extensions<ul> <li>extension_interface</li> <li>functions</li> <li>sklearn<ul> <li>extension</li> </ul> </li> </ul> </li> <li>flows<ul> <li>flow</li> <li>functions</li> </ul> </li> <li>runs<ul> <li>functions</li> <li>run</li> <li>trace</li> </ul> </li> <li>setups<ul> <li>functions</li> <li>setup</li> </ul> </li> <li>study<ul> <li>functions</li> <li>study</li> </ul> </li> <li>tasks<ul> <li>functions</li> <li>split</li> <li>task</li> </ul> </li> <li>testing</li> <li>utils</li> </ul>"},{"location":"reference/__version__/","title":"version","text":"<p>Version information.</p>"},{"location":"reference/_api_calls/","title":"api calls","text":""},{"location":"reference/_api_calls/#openml._api_calls.resolve_env_proxies","title":"<code>resolve_env_proxies(url)</code>","text":"<p>Attempt to find a suitable proxy for this url.</p> <p>Relies on <code>requests</code> internals to remain consistent. To disable this from the environment, please set the enviornment varialbe <code>no_proxy=\"*\"</code>.</p> <p>Parameters:</p> Name Type Description Default <code>url</code> <code>str</code> <p>The url endpoint</p> required <p>Returns:</p> Type Description <code>Optional[str]</code> <p>The proxy url if found, else None</p> Source code in <code>openml/_api_calls.py</code> <pre><code>def resolve_env_proxies(url: str) -&gt; str | None:\n    \"\"\"Attempt to find a suitable proxy for this url.\n\n    Relies on ``requests`` internals to remain consistent. To disable this from the\n    environment, please set the enviornment varialbe ``no_proxy=\"*\"``.\n\n    Parameters\n    ----------\n    url : str\n        The url endpoint\n\n    Returns\n    -------\n    Optional[str]\n        The proxy url if found, else None\n    \"\"\"\n    resolved_proxies = requests.utils.get_environ_proxies(url)\n    return requests.utils.select_proxy(url, resolved_proxies)  # type: ignore\n</code></pre>"},{"location":"reference/base/","title":"base","text":""},{"location":"reference/base/#openml.base.OpenMLBase","title":"<code>OpenMLBase</code>","text":"<p>               Bases: <code>ABC</code></p> <p>Base object for functionality that is shared across entities.</p> Source code in <code>openml/base.py</code> <pre><code>class OpenMLBase(ABC):\n    \"\"\"Base object for functionality that is shared across entities.\"\"\"\n\n    def __repr__(self) -&gt; str:\n        body_fields = self._get_repr_body_fields()\n        return self._apply_repr_template(body_fields)\n\n    @property\n    @abstractmethod\n    def id(self) -&gt; int | None:\n        \"\"\"The id of the entity, it is unique for its entity type.\"\"\"\n\n    @property\n    def openml_url(self) -&gt; str | None:\n        \"\"\"The URL of the object on the server, if it was uploaded, else None.\"\"\"\n        if self.id is None:\n            return None\n        return self.__class__.url_for_id(self.id)\n\n    @classmethod\n    def url_for_id(cls, id_: int) -&gt; str:\n        \"\"\"Return the OpenML URL for the object of the class entity with the given id.\"\"\"\n        # Sample url for a flow: openml.org/f/123\n        return f\"{openml.config.get_server_base_url()}/{cls._entity_letter()}/{id_}\"\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        \"\"\"Return the letter which represents the entity type in urls, e.g. 'f' for flow.\"\"\"\n        # We take advantage of the class naming convention (OpenMLX),\n        # which holds for all entities except studies and tasks, which overwrite this method.\n        return cls.__name__.lower()[len(\"OpenML\") :][0]\n\n    # TODO(eddiebergman): This would be much cleaner as an iterator...\n    @abstractmethod\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str] | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\n\n        Returns\n        -------\n        body_fields : List[Tuple[str, Union[str, int, List[str]]]]\n            A list of (name, value) pairs to display in the body of the __repr__.\n            E.g.: [('metric', 'accuracy'), ('dataset', 'iris')]\n            If value is a List of str, then each item of the list will appear in a separate row.\n        \"\"\"\n        # Should be implemented in the base class.\n\n    def _apply_repr_template(\n        self,\n        body_fields: Iterable[tuple[str, str | int | list[str] | None]],\n    ) -&gt; str:\n        \"\"\"Generates the header and formats the body for string representation of the object.\n\n        Parameters\n        ----------\n        body_fields: List[Tuple[str, str]]\n           A list of (name, value) pairs to display in the body of the __repr__.\n        \"\"\"\n        # We add spaces between capitals, e.g. ClassificationTask -&gt; Classification Task\n        name_with_spaces = re.sub(\n            r\"(\\w)([A-Z])\",\n            r\"\\1 \\2\",\n            self.__class__.__name__[len(\"OpenML\") :],\n        )\n        header_text = f\"OpenML {name_with_spaces}\"\n        header = \"{}\\n{}\\n\".format(header_text, \"=\" * len(header_text))\n\n        _body_fields: list[tuple[str, str | int | list[str]]] = [\n            (k, \"None\" if v is None else v) for k, v in body_fields\n        ]\n        longest_field_name_length = max(len(name) for name, _ in _body_fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _body_fields)\n        return header + body\n\n    @abstractmethod\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\n\n        The return value will be used to create the upload xml file.\n        The xml file must have the tags in exactly the order of the object's xsd.\n        (see https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/).\n\n        Returns\n        -------\n            Thing represented as dict.\n        \"\"\"\n        # Should be implemented in the base class.\n\n    def _to_xml(self) -&gt; str:\n        \"\"\"Generate xml representation of self for upload to server.\"\"\"\n        dict_representation = self._to_dict()\n        xml_representation = xmltodict.unparse(dict_representation, pretty=True)\n\n        # A task may not be uploaded with the xml encoding specification:\n        # &lt;?xml version=\"1.0\" encoding=\"utf-8\"?&gt;\n        _encoding_specification, xml_body = xml_representation.split(\"\\n\", 1)\n        return str(xml_body)\n\n    def _get_file_elements(self) -&gt; openml._api_calls.FILE_ELEMENTS_TYPE:\n        \"\"\"Get file_elements to upload to the server, called during Publish.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        return {}\n\n    @abstractmethod\n    def _parse_publish_response(self, xml_response: dict[str, str]) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n\n    def publish(self) -&gt; OpenMLBase:\n        \"\"\"Publish the object on the OpenML server.\"\"\"\n        file_elements = self._get_file_elements()\n\n        if \"description\" not in file_elements:\n            file_elements[\"description\"] = self._to_xml()\n\n        call = f\"{_get_rest_api_type_alias(self)}/\"\n        response_text = openml._api_calls._perform_api_call(\n            call,\n            \"post\",\n            file_elements=file_elements,\n        )\n        xml_response = xmltodict.parse(response_text)\n\n        self._parse_publish_response(xml_response)\n        return self\n\n    def open_in_browser(self) -&gt; None:\n        \"\"\"Opens the OpenML web page corresponding to this object in your default browser.\"\"\"\n        if self.openml_url is None:\n            raise ValueError(\n                \"Cannot open element on OpenML.org when attribute `openml_url` is `None`\",\n            )\n\n        webbrowser.open(self.openml_url)\n\n    def push_tag(self, tag: str) -&gt; None:\n        \"\"\"Annotates this entity with a tag on the server.\n\n        Parameters\n        ----------\n        tag : str\n            Tag to attach to the flow.\n        \"\"\"\n        _tag_openml_base(self, tag)\n\n    def remove_tag(self, tag: str) -&gt; None:\n        \"\"\"Removes a tag from this entity on the server.\n\n        Parameters\n        ----------\n        tag : str\n            Tag to attach to the flow.\n        \"\"\"\n        _tag_openml_base(self, tag, untag=True)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.id","title":"<code>id: int | None</code>  <code>abstractmethod</code> <code>property</code>","text":"<p>The id of the entity, it is unique for its entity type.</p>"},{"location":"reference/base/#openml.base.OpenMLBase.openml_url","title":"<code>openml_url: str | None</code>  <code>property</code>","text":"<p>The URL of the object on the server, if it was uploaded, else None.</p>"},{"location":"reference/base/#openml.base.OpenMLBase.open_in_browser","title":"<code>open_in_browser()</code>","text":"<p>Opens the OpenML web page corresponding to this object in your default browser.</p> Source code in <code>openml/base.py</code> <pre><code>def open_in_browser(self) -&gt; None:\n    \"\"\"Opens the OpenML web page corresponding to this object in your default browser.\"\"\"\n    if self.openml_url is None:\n        raise ValueError(\n            \"Cannot open element on OpenML.org when attribute `openml_url` is `None`\",\n        )\n\n    webbrowser.open(self.openml_url)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.publish","title":"<code>publish()</code>","text":"<p>Publish the object on the OpenML server.</p> Source code in <code>openml/base.py</code> <pre><code>def publish(self) -&gt; OpenMLBase:\n    \"\"\"Publish the object on the OpenML server.\"\"\"\n    file_elements = self._get_file_elements()\n\n    if \"description\" not in file_elements:\n        file_elements[\"description\"] = self._to_xml()\n\n    call = f\"{_get_rest_api_type_alias(self)}/\"\n    response_text = openml._api_calls._perform_api_call(\n        call,\n        \"post\",\n        file_elements=file_elements,\n    )\n    xml_response = xmltodict.parse(response_text)\n\n    self._parse_publish_response(xml_response)\n    return self\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.push_tag","title":"<code>push_tag(tag)</code>","text":"<p>Annotates this entity with a tag on the server.</p> <p>Parameters:</p> Name Type Description Default <code>tag</code> <code>str</code> <p>Tag to attach to the flow.</p> required Source code in <code>openml/base.py</code> <pre><code>def push_tag(self, tag: str) -&gt; None:\n    \"\"\"Annotates this entity with a tag on the server.\n\n    Parameters\n    ----------\n    tag : str\n        Tag to attach to the flow.\n    \"\"\"\n    _tag_openml_base(self, tag)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.remove_tag","title":"<code>remove_tag(tag)</code>","text":"<p>Removes a tag from this entity on the server.</p> <p>Parameters:</p> Name Type Description Default <code>tag</code> <code>str</code> <p>Tag to attach to the flow.</p> required Source code in <code>openml/base.py</code> <pre><code>def remove_tag(self, tag: str) -&gt; None:\n    \"\"\"Removes a tag from this entity on the server.\n\n    Parameters\n    ----------\n    tag : str\n        Tag to attach to the flow.\n    \"\"\"\n    _tag_openml_base(self, tag, untag=True)\n</code></pre>"},{"location":"reference/base/#openml.base.OpenMLBase.url_for_id","title":"<code>url_for_id(id_)</code>  <code>classmethod</code>","text":"<p>Return the OpenML URL for the object of the class entity with the given id.</p> Source code in <code>openml/base.py</code> <pre><code>@classmethod\ndef url_for_id(cls, id_: int) -&gt; str:\n    \"\"\"Return the OpenML URL for the object of the class entity with the given id.\"\"\"\n    # Sample url for a flow: openml.org/f/123\n    return f\"{openml.config.get_server_base_url()}/{cls._entity_letter()}/{id_}\"\n</code></pre>"},{"location":"reference/cli/","title":"cli","text":"<p>\"Command Line Interface for <code>openml</code> to configure its settings.</p>"},{"location":"reference/cli/#openml.cli.configure","title":"<code>configure(args)</code>","text":"<p>Calls the right submenu(s) to edit <code>args.field</code> in the configuration file.</p> Source code in <code>openml/cli.py</code> <pre><code>def configure(args: argparse.Namespace) -&gt; None:\n    \"\"\"Calls the right submenu(s) to edit `args.field` in the configuration file.\"\"\"\n    set_functions = {\n        \"apikey\": configure_apikey,\n        \"server\": configure_server,\n        \"cachedir\": configure_cachedir,\n        \"retry_policy\": configure_retry_policy,\n        \"connection_n_retries\": configure_connection_n_retries,\n        \"avoid_duplicate_runs\": configure_avoid_duplicate_runs,\n        \"verbosity\": configure_verbosity,\n    }\n\n    def not_supported_yet(_: str) -&gt; None:\n        print(f\"Setting '{args.field}' is not supported yet.\")\n\n    if args.field not in [\"all\", \"none\"]:\n        set_functions.get(args.field, not_supported_yet)(args.value)\n    else:\n        if args.value is not None:\n            print(f\"Can not set value ('{args.value}') when field is specified as '{args.field}'.\")\n            sys.exit()\n        print_configuration()\n\n    if args.field == \"all\":\n        for set_field_function in set_functions.values():\n            set_field_function(args.value)\n</code></pre>"},{"location":"reference/cli/#openml.cli.configure_field","title":"<code>configure_field(field, value, check_with_message, intro_message, input_message, sanitize=None)</code>","text":"<p>Configure <code>field</code> with <code>value</code>. If <code>value</code> is None ask the user for input.</p> <p><code>value</code> and user input are first corrected/auto-completed with <code>convert_value</code> if provided, then validated with <code>check_with_message</code> function. If the user input a wrong value in interactive mode, the user gets to input a new value. The new valid value is saved in the openml configuration file. In case an invalid <code>value</code> is supplied directly (non-interactive), no changes are made.</p> <p>Parameters:</p> Name Type Description Default <code>field</code> <code>str</code> <p>Field to set.</p> required <code>value</code> <code>None | str</code> <p>Value to field to. If <code>None</code> will ask user for input.</p> required <code>check_with_message</code> <code>Callable[[str], str]</code> <p>Function which validates <code>value</code> or user input, and returns either an error message if it is invalid, or a False-like value if <code>value</code> is valid.</p> required <code>intro_message</code> <code>str</code> <p>Message that is printed once if user input is requested (e.g. instructions).</p> required <code>input_message</code> <code>str</code> <p>Message that comes with the input prompt.</p> required <code>sanitize</code> <code>Callable[[str], str] | None</code> <p>A function to convert user input to 'more acceptable' input, e.g. for auto-complete. If no correction of user input is possible, return the original value. If no function is provided, don't attempt to correct/auto-complete input.</p> <code>None</code> Source code in <code>openml/cli.py</code> <pre><code>def configure_field(  # noqa: PLR0913\n    field: str,\n    value: None | str,\n    check_with_message: Callable[[str], str],\n    intro_message: str,\n    input_message: str,\n    sanitize: Callable[[str], str] | None = None,\n) -&gt; None:\n    \"\"\"Configure `field` with `value`. If `value` is None ask the user for input.\n\n    `value` and user input are first corrected/auto-completed with `convert_value` if provided,\n    then validated with `check_with_message` function.\n    If the user input a wrong value in interactive mode, the user gets to input a new value.\n    The new valid value is saved in the openml configuration file.\n    In case an invalid `value` is supplied directly (non-interactive), no changes are made.\n\n    Parameters\n    ----------\n    field: str\n        Field to set.\n    value: str, None\n        Value to field to. If `None` will ask user for input.\n    check_with_message: Callable[[str], str]\n        Function which validates `value` or user input, and returns either an error message if it\n        is invalid, or a False-like value if `value` is valid.\n    intro_message: str\n        Message that is printed once if user input is requested (e.g. instructions).\n    input_message: str\n        Message that comes with the input prompt.\n    sanitize: Union[Callable[[str], str], None]\n        A function to convert user input to 'more acceptable' input, e.g. for auto-complete.\n        If no correction of user input is possible, return the original value.\n        If no function is provided, don't attempt to correct/auto-complete input.\n    \"\"\"\n    if value is not None:\n        if sanitize:\n            value = sanitize(value)\n        malformed_input = check_with_message(value)\n        if malformed_input:\n            print(malformed_input)\n            sys.exit()\n    else:\n        print(intro_message)\n        value = wait_until_valid_input(\n            prompt=input_message,\n            check=check_with_message,\n            sanitize=sanitize,\n        )\n    verbose_set(field, value)\n</code></pre>"},{"location":"reference/cli/#openml.cli.wait_until_valid_input","title":"<code>wait_until_valid_input(prompt, check, sanitize)</code>","text":"<p>Asks <code>prompt</code> until an input is received which returns True for <code>check</code>.</p> <p>Parameters:</p> Name Type Description Default <code>prompt</code> <code>str</code> <p>message to display</p> required <code>check</code> <code>Callable[[str], str]</code> <p>function to call with the given input, that provides an error message if the input is not valid otherwise, and False-like otherwise.</p> required <code>sanitize</code> <code>Callable[[str], str] | None</code> <p>A function which attempts to sanitize the user input (e.g. auto-complete).</p> required <p>Returns:</p> Type Description <code>valid input</code> Source code in <code>openml/cli.py</code> <pre><code>def wait_until_valid_input(\n    prompt: str,\n    check: Callable[[str], str],\n    sanitize: Callable[[str], str] | None,\n) -&gt; str:\n    \"\"\"Asks `prompt` until an input is received which returns True for `check`.\n\n    Parameters\n    ----------\n    prompt: str\n        message to display\n    check: Callable[[str], str]\n        function to call with the given input, that provides an error message if the input is not\n        valid otherwise, and False-like otherwise.\n    sanitize: Callable[[str], str], optional\n        A function which attempts to sanitize the user input (e.g. auto-complete).\n\n    Returns\n    -------\n    valid input\n\n    \"\"\"\n    while True:\n        response = input(prompt)\n        if sanitize:\n            response = sanitize(response)\n        error_message = check(response)\n        if error_message:\n            print(error_message, end=\"\\n\\n\")\n        else:\n            return response\n</code></pre>"},{"location":"reference/config/","title":"config","text":"<p>Store module level information like the API key, cache directory and the server</p>"},{"location":"reference/config/#openml.config.ConfigurationForExamples","title":"<code>ConfigurationForExamples</code>","text":"<p>Allows easy switching to and from a test configuration, used for examples.</p> Source code in <code>openml/config.py</code> <pre><code>class ConfigurationForExamples:\n    \"\"\"Allows easy switching to and from a test configuration, used for examples.\"\"\"\n\n    _last_used_server = None\n    _last_used_key = None\n    _start_last_called = False\n    _test_server = \"https://test.openml.org/api/v1/xml\"\n    _test_apikey = \"c0c42819af31e706efe1f4b88c23c6c1\"\n\n    @classmethod\n    def start_using_configuration_for_example(cls) -&gt; None:\n        \"\"\"Sets the configuration to connect to the test server with valid apikey.\n\n        To configuration as was before this call is stored, and can be recovered\n        by using the `stop_use_example_configuration` method.\n        \"\"\"\n        global server  # noqa: PLW0603\n        global apikey  # noqa: PLW0603\n\n        if cls._start_last_called and server == cls._test_server and apikey == cls._test_apikey:\n            # Method is called more than once in a row without modifying the server or apikey.\n            # We don't want to save the current test configuration as a last used configuration.\n            return\n\n        cls._last_used_server = server\n        cls._last_used_key = apikey\n        cls._start_last_called = True\n\n        # Test server key for examples\n        server = cls._test_server\n        apikey = cls._test_apikey\n        warnings.warn(\n            f\"Switching to the test server {server} to not upload results to the live server. \"\n            \"Using the test server may result in reduced performance of the API!\",\n            stacklevel=2,\n        )\n\n    @classmethod\n    def stop_using_configuration_for_example(cls) -&gt; None:\n        \"\"\"Return to configuration as it was before `start_use_example_configuration`.\"\"\"\n        if not cls._start_last_called:\n            # We don't want to allow this because it will (likely) result in the `server` and\n            # `apikey` variables being set to None.\n            raise RuntimeError(\n                \"`stop_use_example_configuration` called without a saved config.\"\n                \"`start_use_example_configuration` must be called first.\",\n            )\n\n        global server  # noqa: PLW0603\n        global apikey  # noqa: PLW0603\n\n        server = cast(str, cls._last_used_server)\n        apikey = cast(str, cls._last_used_key)\n        cls._start_last_called = False\n</code></pre>"},{"location":"reference/config/#openml.config.ConfigurationForExamples.start_using_configuration_for_example","title":"<code>start_using_configuration_for_example()</code>  <code>classmethod</code>","text":"<p>Sets the configuration to connect to the test server with valid apikey.</p> <p>To configuration as was before this call is stored, and can be recovered by using the <code>stop_use_example_configuration</code> method.</p> Source code in <code>openml/config.py</code> <pre><code>@classmethod\ndef start_using_configuration_for_example(cls) -&gt; None:\n    \"\"\"Sets the configuration to connect to the test server with valid apikey.\n\n    To configuration as was before this call is stored, and can be recovered\n    by using the `stop_use_example_configuration` method.\n    \"\"\"\n    global server  # noqa: PLW0603\n    global apikey  # noqa: PLW0603\n\n    if cls._start_last_called and server == cls._test_server and apikey == cls._test_apikey:\n        # Method is called more than once in a row without modifying the server or apikey.\n        # We don't want to save the current test configuration as a last used configuration.\n        return\n\n    cls._last_used_server = server\n    cls._last_used_key = apikey\n    cls._start_last_called = True\n\n    # Test server key for examples\n    server = cls._test_server\n    apikey = cls._test_apikey\n    warnings.warn(\n        f\"Switching to the test server {server} to not upload results to the live server. \"\n        \"Using the test server may result in reduced performance of the API!\",\n        stacklevel=2,\n    )\n</code></pre>"},{"location":"reference/config/#openml.config.ConfigurationForExamples.stop_using_configuration_for_example","title":"<code>stop_using_configuration_for_example()</code>  <code>classmethod</code>","text":"<p>Return to configuration as it was before <code>start_use_example_configuration</code>.</p> Source code in <code>openml/config.py</code> <pre><code>@classmethod\ndef stop_using_configuration_for_example(cls) -&gt; None:\n    \"\"\"Return to configuration as it was before `start_use_example_configuration`.\"\"\"\n    if not cls._start_last_called:\n        # We don't want to allow this because it will (likely) result in the `server` and\n        # `apikey` variables being set to None.\n        raise RuntimeError(\n            \"`stop_use_example_configuration` called without a saved config.\"\n            \"`start_use_example_configuration` must be called first.\",\n        )\n\n    global server  # noqa: PLW0603\n    global apikey  # noqa: PLW0603\n\n    server = cast(str, cls._last_used_server)\n    apikey = cast(str, cls._last_used_key)\n    cls._start_last_called = False\n</code></pre>"},{"location":"reference/config/#openml.config.get_cache_directory","title":"<code>get_cache_directory()</code>","text":"<p>Get the current cache directory.</p> <p>This gets the cache directory for the current server relative to the root cache directory that can be set via <code>set_root_cache_directory()</code>. The cache directory is the <code>root_cache_directory</code> with additional information on which subdirectory to use based on the server name. By default it is <code>root_cache_directory / org / openml / www</code> for the standard OpenML.org server and is defined as <code>root_cache_directory / top-level domain / second-level domain / hostname</code> ```</p>"},{"location":"reference/config/#openml.config.get_cache_directory--returns","title":"Returns","text":"<p>cachedir : string     The current cache directory.</p> Source code in <code>openml/config.py</code> <pre><code>def get_cache_directory() -&gt; str:\n    \"\"\"Get the current cache directory.\n\n    This gets the cache directory for the current server relative\n    to the root cache directory that can be set via\n    ``set_root_cache_directory()``. The cache directory is the\n    ``root_cache_directory`` with additional information on which\n    subdirectory to use based on the server name. By default it is\n    ``root_cache_directory / org / openml / www`` for the standard\n    OpenML.org server and is defined as\n    ``root_cache_directory / top-level domain / second-level domain /\n    hostname``\n    ```\n\n    Returns\n    -------\n    cachedir : string\n        The current cache directory.\n\n    \"\"\"\n    url_suffix = urlparse(server).netloc\n    reversed_url_suffix = os.sep.join(url_suffix.split(\".\")[::-1])  # noqa: PTH118\n    return os.path.join(_root_cache_directory, reversed_url_suffix)  # noqa: PTH118\n</code></pre>"},{"location":"reference/config/#openml.config.get_server_base_url","title":"<code>get_server_base_url()</code>","text":"<p>Return the base URL of the currently configured server.</p> <p>Turns <code>\"https://www.openml.org/api/v1/xml\"</code> in <code>\"https://www.openml.org/\"</code></p> <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/config.py</code> <pre><code>def get_server_base_url() -&gt; str:\n    \"\"\"Return the base URL of the currently configured server.\n\n    Turns ``\"https://www.openml.org/api/v1/xml\"`` in ``\"https://www.openml.org/\"``\n\n    Returns\n    -------\n    str\n    \"\"\"\n    return server.split(\"/api\")[0]\n</code></pre>"},{"location":"reference/config/#openml.config.set_console_log_level","title":"<code>set_console_log_level(console_output_level)</code>","text":"<p>Set console output to the desired level and register it with openml logger if needed.</p> Source code in <code>openml/config.py</code> <pre><code>def set_console_log_level(console_output_level: int) -&gt; None:\n    \"\"\"Set console output to the desired level and register it with openml logger if needed.\"\"\"\n    global console_handler  # noqa: PLW0602\n    assert console_handler is not None\n    _set_level_register_and_store(console_handler, console_output_level)\n</code></pre>"},{"location":"reference/config/#openml.config.set_field_in_config_file","title":"<code>set_field_in_config_file(field, value)</code>","text":"<p>Overwrites the <code>field</code> in the configuration file with the new <code>value</code>.</p> Source code in <code>openml/config.py</code> <pre><code>def set_field_in_config_file(field: str, value: Any) -&gt; None:\n    \"\"\"Overwrites the `field` in the configuration file with the new `value`.\"\"\"\n    if field not in _defaults:\n        raise ValueError(f\"Field '{field}' is not valid and must be one of '{_defaults.keys()}'.\")\n\n    # TODO(eddiebergman): This use of globals has gone too far\n    globals()[field] = value\n    config_file = determine_config_file_path()\n    config = _parse_config(config_file)\n    with config_file.open(\"w\") as fh:\n        for f in _defaults:\n            # We can't blindly set all values based on globals() because when the user\n            # sets it through config.FIELD it should not be stored to file.\n            # There doesn't seem to be a way to avoid writing defaults to file with configparser,\n            # because it is impossible to distinguish from an explicitly set value that matches\n            # the default value, to one that was set to its default because it was omitted.\n            value = config.get(\"FAKE_SECTION\", f)  # type: ignore\n            if f == field:\n                value = globals()[f]\n            fh.write(f\"{f} = {value}\\n\")\n</code></pre>"},{"location":"reference/config/#openml.config.set_file_log_level","title":"<code>set_file_log_level(file_output_level)</code>","text":"<p>Set file output to the desired level and register it with openml logger if needed.</p> Source code in <code>openml/config.py</code> <pre><code>def set_file_log_level(file_output_level: int) -&gt; None:\n    \"\"\"Set file output to the desired level and register it with openml logger if needed.\"\"\"\n    global file_handler  # noqa: PLW0602\n    assert file_handler is not None\n    _set_level_register_and_store(file_handler, file_output_level)\n</code></pre>"},{"location":"reference/config/#openml.config.set_root_cache_directory","title":"<code>set_root_cache_directory(root_cache_directory)</code>","text":"<p>Set module-wide base cache directory.</p> <p>Sets the root cache directory, wherin the cache directories are created to store content from different OpenML servers. For example, by default, cached data for the standard OpenML.org server is stored at <code>root_cache_directory / org / openml / www</code>, and the general pattern is <code>root_cache_directory / top-level domain / second-level domain / hostname</code>.</p> <p>Parameters:</p> Name Type Description Default <code>root_cache_directory</code> <code>string</code> <p>Path to use as cache directory.</p> required See Also <p>get_cache_directory</p> Source code in <code>openml/config.py</code> <pre><code>def set_root_cache_directory(root_cache_directory: str | Path) -&gt; None:\n    \"\"\"Set module-wide base cache directory.\n\n    Sets the root cache directory, wherin the cache directories are\n    created to store content from different OpenML servers. For example,\n    by default, cached data for the standard OpenML.org server is stored\n    at ``root_cache_directory / org / openml / www``, and the general\n    pattern is ``root_cache_directory / top-level domain / second-level\n    domain / hostname``.\n\n    Parameters\n    ----------\n    root_cache_directory : string\n         Path to use as cache directory.\n\n    See Also\n    --------\n    get_cache_directory\n    \"\"\"\n    global _root_cache_directory  # noqa: PLW0603\n    _root_cache_directory = Path(root_cache_directory)\n</code></pre>"},{"location":"reference/exceptions/","title":"exceptions","text":""},{"location":"reference/exceptions/#openml.exceptions.ObjectNotPublishedError","title":"<code>ObjectNotPublishedError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Indicates an object has not been published yet.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class ObjectNotPublishedError(PyOpenMLError):\n    \"\"\"Indicates an object has not been published yet.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLCacheException","title":"<code>OpenMLCacheException</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Dataset / task etc not found in cache</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLCacheException(PyOpenMLError):  # noqa: N818\n    \"\"\"Dataset / task etc not found in cache\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLHashException","title":"<code>OpenMLHashException</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Locally computed hash is different than hash announced by the server.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLHashException(PyOpenMLError):  # noqa: N818\n    \"\"\"Locally computed hash is different than hash announced by the server.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLNotAuthorizedError","title":"<code>OpenMLNotAuthorizedError</code>","text":"<p>               Bases: <code>OpenMLServerError</code></p> <p>Indicates an authenticated user is not authorized to execute the requested action.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLNotAuthorizedError(OpenMLServerError):\n    \"\"\"Indicates an authenticated user is not authorized to execute the requested action.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLPrivateDatasetError","title":"<code>OpenMLPrivateDatasetError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Exception thrown when the user has no rights to access the dataset.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLPrivateDatasetError(PyOpenMLError):\n    \"\"\"Exception thrown when the user has no rights to access the dataset.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLRunsExistError","title":"<code>OpenMLRunsExistError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>Indicates run(s) already exists on the server when they should not be duplicated.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLRunsExistError(PyOpenMLError):\n    \"\"\"Indicates run(s) already exists on the server when they should not be duplicated.\"\"\"\n\n    def __init__(self, run_ids: set[int], message: str) -&gt; None:\n        if len(run_ids) &lt; 1:\n            raise ValueError(\"Set of run ids must be non-empty.\")\n        self.run_ids = run_ids\n        super().__init__(message)\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLServerError","title":"<code>OpenMLServerError</code>","text":"<p>               Bases: <code>PyOpenMLError</code></p> <p>class for when something is really wrong on the server (result did not parse to dict), contains unparsed error.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLServerError(PyOpenMLError):\n    \"\"\"class for when something is really wrong on the server\n    (result did not parse to dict), contains unparsed error.\n    \"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLServerException","title":"<code>OpenMLServerException</code>","text":"<p>               Bases: <code>OpenMLServerError</code></p> <p>exception for when the result of the server was not 200 (e.g., listing call w/o results).</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLServerException(OpenMLServerError):  # noqa: N818\n    \"\"\"exception for when the result of the server was\n    not 200 (e.g., listing call w/o results).\n    \"\"\"\n\n    # Code needs to be optional to allow the exception to be picklable:\n    # https://stackoverflow.com/questions/16244923/how-to-make-a-custom-exception-class-with-multiple-init-args-pickleable  # noqa: E501\n    def __init__(self, message: str, code: int | None = None, url: str | None = None):\n        self.message = message\n        self.code = code\n        self.url = url\n        super().__init__(message)\n\n    def __str__(self) -&gt; str:\n        return f\"{self.url} returned code {self.code}: {self.message}\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.OpenMLServerNoResult","title":"<code>OpenMLServerNoResult</code>","text":"<p>               Bases: <code>OpenMLServerException</code></p> <p>Exception for when the result of the server is empty.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class OpenMLServerNoResult(OpenMLServerException):\n    \"\"\"Exception for when the result of the server is empty.\"\"\"\n</code></pre>"},{"location":"reference/exceptions/#openml.exceptions.PyOpenMLError","title":"<code>PyOpenMLError</code>","text":"<p>               Bases: <code>Exception</code></p> <p>Base class for all exceptions in OpenML-Python.</p> Source code in <code>openml/exceptions.py</code> <pre><code>class PyOpenMLError(Exception):\n    \"\"\"Base class for all exceptions in OpenML-Python.\"\"\"\n\n    def __init__(self, message: str):\n        self.message = message\n        super().__init__(message)\n</code></pre>"},{"location":"reference/testing/","title":"testing","text":""},{"location":"reference/testing/#openml.testing.CustomImputer","title":"<code>CustomImputer</code>","text":"<p>               Bases: <code>Imputer</code></p> <p>Duplicate class alias for sklearn's SimpleImputer</p> <p>Helps bypass the sklearn extension duplicate operation check</p> Source code in <code>openml/testing.py</code> <pre><code>class CustomImputer(SimpleImputer):\n    \"\"\"Duplicate class alias for sklearn's SimpleImputer\n\n    Helps bypass the sklearn extension duplicate operation check\n    \"\"\"\n</code></pre>"},{"location":"reference/testing/#openml.testing.TestBase","title":"<code>TestBase</code>","text":"<p>               Bases: <code>TestCase</code></p> <p>Base class for tests</p> Note <p>Currently hard-codes a read-write key. Hopefully soon allows using a test server, not the production server.</p> Source code in <code>openml/testing.py</code> <pre><code>class TestBase(unittest.TestCase):\n    \"\"\"Base class for tests\n\n    Note\n    ----\n    Currently hard-codes a read-write key.\n    Hopefully soon allows using a test server, not the production server.\n    \"\"\"\n\n    # TODO: This could be made more explcit with a TypedDict instead of list[str | int]\n    publish_tracker: ClassVar[dict[str, list[str | int]]] = {\n        \"run\": [],\n        \"data\": [],\n        \"flow\": [],\n        \"task\": [],\n        \"study\": [],\n        \"user\": [],\n    }\n    flow_name_tracker: ClassVar[list[str]] = []\n    test_server = \"https://test.openml.org/api/v1/xml\"\n    # amueller's read/write key that he will throw away later\n    apikey = \"610344db6388d9ba34f6db45a3cf71de\"\n\n    # creating logger for tracking files uploaded to test server\n    logger = logging.getLogger(\"unit_tests_published_entities\")\n    logger.setLevel(logging.DEBUG)\n\n    def setUp(self, n_levels: int = 1) -&gt; None:\n        \"\"\"Setup variables and temporary directories.\n\n        In particular, this methods:\n\n        * creates a temporary working directory\n        * figures out a path to a few static test files\n        * set the default server to be the test server\n        * set a static API key for the test server\n        * increases the maximal number of retries\n\n        Parameters\n        ----------\n        n_levels : int\n            Number of nested directories the test is in. Necessary to resolve the path to the\n            ``files`` directory, which is located directly under the ``tests`` directory.\n        \"\"\"\n        # This cache directory is checked in to git to simulate a populated\n        # cache\n        self.maxDiff = None\n        abspath_this_file = Path(inspect.getfile(self.__class__)).absolute()\n        static_cache_dir = abspath_this_file.parent\n        for _ in range(n_levels):\n            static_cache_dir = static_cache_dir.parent.absolute()\n\n        content = os.listdir(static_cache_dir)\n        if \"files\" in content:\n            static_cache_dir = static_cache_dir / \"files\"\n        else:\n            raise ValueError(\n                f\"Cannot find test cache dir, expected it to be {static_cache_dir}!\",\n            )\n\n        self.static_cache_dir = static_cache_dir\n        self.cwd = Path.cwd()\n        workdir = Path(__file__).parent.absolute()\n        tmp_dir_name = self.id()\n        self.workdir = workdir / tmp_dir_name\n        shutil.rmtree(self.workdir, ignore_errors=True)\n\n        self.workdir.mkdir(exist_ok=True)\n        os.chdir(self.workdir)\n\n        self.cached = True\n        openml.config.apikey = TestBase.apikey\n        self.production_server = \"https://openml.org/api/v1/xml\"\n        openml.config.server = TestBase.test_server\n        openml.config.avoid_duplicate_runs = False\n        openml.config.set_root_cache_directory(str(self.workdir))\n\n        # Increase the number of retries to avoid spurious server failures\n        self.retry_policy = openml.config.retry_policy\n        self.connection_n_retries = openml.config.connection_n_retries\n        openml.config.set_retry_policy(\"robot\", n_retries=20)\n\n    def tearDown(self) -&gt; None:\n        \"\"\"Tear down the test\"\"\"\n        os.chdir(self.cwd)\n        try:\n            shutil.rmtree(self.workdir)\n        except PermissionError as e:\n            if os.name != \"nt\":\n                # one of the files may still be used by another process\n                raise e\n\n        openml.config.server = self.production_server\n        openml.config.connection_n_retries = self.connection_n_retries\n        openml.config.retry_policy = self.retry_policy\n\n    @classmethod\n    def _mark_entity_for_removal(\n        cls,\n        entity_type: str,\n        entity_id: int,\n        entity_name: str | None = None,\n    ) -&gt; None:\n        \"\"\"Static record of entities uploaded to test server\n\n        Dictionary of lists where the keys are 'entity_type'.\n        Each such dictionary is a list of integer IDs.\n        For entity_type='flow', each list element is a tuple\n        of the form (Flow ID, Flow Name).\n        \"\"\"\n        if entity_type not in TestBase.publish_tracker:\n            TestBase.publish_tracker[entity_type] = [entity_id]\n        else:\n            TestBase.publish_tracker[entity_type].append(entity_id)\n        if isinstance(entity_type, openml.flows.OpenMLFlow):\n            assert entity_name is not None\n            cls.flow_name_tracker.append(entity_name)\n\n    @classmethod\n    def _delete_entity_from_tracker(cls, entity_type: str, entity: int) -&gt; None:\n        \"\"\"Deletes entity records from the static file_tracker\n\n        Given an entity type and corresponding ID, deletes all entries, including\n        duplicate entries of the ID for the entity type.\n        \"\"\"\n        if entity_type in TestBase.publish_tracker:\n            # removes duplicate entries\n            TestBase.publish_tracker[entity_type] = list(set(TestBase.publish_tracker[entity_type]))\n            if entity_type == \"flow\":\n                delete_index = next(\n                    i\n                    for i, (id_, _) in enumerate(\n                        zip(TestBase.publish_tracker[entity_type], TestBase.flow_name_tracker),\n                    )\n                    if id_ == entity\n                )\n            else:\n                delete_index = next(\n                    i\n                    for i, id_ in enumerate(TestBase.publish_tracker[entity_type])\n                    if id_ == entity\n                )\n            TestBase.publish_tracker[entity_type].pop(delete_index)\n\n    def _get_sentinel(self, sentinel: str | None = None) -&gt; str:\n        if sentinel is None:\n            # Create a unique prefix for the flow. Necessary because the flow\n            # is identified by its name and external version online. Having a\n            # unique name allows us to publish the same flow in each test run.\n            md5 = hashlib.md5()  # noqa: S324\n            md5.update(str(time.time()).encode(\"utf-8\"))\n            md5.update(str(os.getpid()).encode(\"utf-8\"))\n            sentinel = md5.hexdigest()[:10]\n            sentinel = \"TEST%s\" % sentinel\n        return sentinel\n\n    def _add_sentinel_to_flow_name(\n        self,\n        flow: openml.flows.OpenMLFlow,\n        sentinel: str | None = None,\n    ) -&gt; tuple[openml.flows.OpenMLFlow, str]:\n        sentinel = self._get_sentinel(sentinel=sentinel)\n        flows_to_visit = []\n        flows_to_visit.append(flow)\n        while len(flows_to_visit) &gt; 0:\n            current_flow = flows_to_visit.pop()\n            current_flow.name = f\"{sentinel}{current_flow.name}\"\n            for subflow in current_flow.components.values():\n                flows_to_visit.append(subflow)\n\n        return flow, sentinel\n\n    def _check_dataset(self, dataset: dict[str, str | int]) -&gt; None:\n        _check_dataset(dataset)\n        assert isinstance(dataset, dict)\n        assert len(dataset) &gt;= 2\n        assert \"did\" in dataset\n        assert isinstance(dataset[\"did\"], int)\n        assert \"status\" in dataset\n        assert isinstance(dataset[\"status\"], str)\n        assert dataset[\"status\"] in [\"in_preparation\", \"active\", \"deactivated\"]\n\n    def _check_fold_timing_evaluations(  # noqa: PLR0913\n        self,\n        fold_evaluations: dict[str, dict[int, dict[int, float]]],\n        num_repeats: int,\n        num_folds: int,\n        *,\n        max_time_allowed: float = 60000.0,\n        task_type: TaskType = TaskType.SUPERVISED_CLASSIFICATION,\n        check_scores: bool = True,\n    ) -&gt; None:\n        \"\"\"\n        Checks whether the right timing measures are attached to the run\n        (before upload). Test is only performed for versions &gt;= Python3.3\n\n        In case of check_n_jobs(clf) == false, please do not perform this\n        check (check this condition outside of this function. )\n        default max_time_allowed (per fold, in milli seconds) = 1 minute,\n        quite pessimistic\n        \"\"\"\n        # a dict mapping from openml measure to a tuple with the minimum and\n        # maximum allowed value\n        check_measures = {\n            # should take at least one millisecond (?)\n            \"usercpu_time_millis_testing\": (0, max_time_allowed),\n            \"usercpu_time_millis_training\": (0, max_time_allowed),\n            \"usercpu_time_millis\": (0, max_time_allowed),\n            \"wall_clock_time_millis_training\": (0, max_time_allowed),\n            \"wall_clock_time_millis_testing\": (0, max_time_allowed),\n            \"wall_clock_time_millis\": (0, max_time_allowed),\n        }\n\n        if check_scores:\n            if task_type in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n                check_measures[\"predictive_accuracy\"] = (0, 1.0)\n            elif task_type == TaskType.SUPERVISED_REGRESSION:\n                check_measures[\"mean_absolute_error\"] = (0, float(\"inf\"))\n\n        assert isinstance(fold_evaluations, dict)\n        assert set(fold_evaluations.keys()) == set(check_measures.keys())\n\n        for measure in check_measures:\n            if measure in fold_evaluations:\n                num_rep_entrees = len(fold_evaluations[measure])\n                assert num_rep_entrees == num_repeats\n                min_val = check_measures[measure][0]\n                max_val = check_measures[measure][1]\n                for rep in range(num_rep_entrees):\n                    num_fold_entrees = len(fold_evaluations[measure][rep])\n                    assert num_fold_entrees == num_folds\n                    for fold in range(num_fold_entrees):\n                        evaluation = fold_evaluations[measure][rep][fold]\n                        assert isinstance(evaluation, float)\n                        assert evaluation &gt;= min_val\n                        assert evaluation &lt;= max_val\n</code></pre>"},{"location":"reference/testing/#openml.testing.TestBase.setUp","title":"<code>setUp(n_levels=1)</code>","text":"<p>Setup variables and temporary directories.</p> <p>In particular, this methods:</p> <ul> <li>creates a temporary working directory</li> <li>figures out a path to a few static test files</li> <li>set the default server to be the test server</li> <li>set a static API key for the test server</li> <li>increases the maximal number of retries</li> </ul> <p>Parameters:</p> Name Type Description Default <code>n_levels</code> <code>int</code> <p>Number of nested directories the test is in. Necessary to resolve the path to the <code>files</code> directory, which is located directly under the <code>tests</code> directory.</p> <code>1</code> Source code in <code>openml/testing.py</code> <pre><code>def setUp(self, n_levels: int = 1) -&gt; None:\n    \"\"\"Setup variables and temporary directories.\n\n    In particular, this methods:\n\n    * creates a temporary working directory\n    * figures out a path to a few static test files\n    * set the default server to be the test server\n    * set a static API key for the test server\n    * increases the maximal number of retries\n\n    Parameters\n    ----------\n    n_levels : int\n        Number of nested directories the test is in. Necessary to resolve the path to the\n        ``files`` directory, which is located directly under the ``tests`` directory.\n    \"\"\"\n    # This cache directory is checked in to git to simulate a populated\n    # cache\n    self.maxDiff = None\n    abspath_this_file = Path(inspect.getfile(self.__class__)).absolute()\n    static_cache_dir = abspath_this_file.parent\n    for _ in range(n_levels):\n        static_cache_dir = static_cache_dir.parent.absolute()\n\n    content = os.listdir(static_cache_dir)\n    if \"files\" in content:\n        static_cache_dir = static_cache_dir / \"files\"\n    else:\n        raise ValueError(\n            f\"Cannot find test cache dir, expected it to be {static_cache_dir}!\",\n        )\n\n    self.static_cache_dir = static_cache_dir\n    self.cwd = Path.cwd()\n    workdir = Path(__file__).parent.absolute()\n    tmp_dir_name = self.id()\n    self.workdir = workdir / tmp_dir_name\n    shutil.rmtree(self.workdir, ignore_errors=True)\n\n    self.workdir.mkdir(exist_ok=True)\n    os.chdir(self.workdir)\n\n    self.cached = True\n    openml.config.apikey = TestBase.apikey\n    self.production_server = \"https://openml.org/api/v1/xml\"\n    openml.config.server = TestBase.test_server\n    openml.config.avoid_duplicate_runs = False\n    openml.config.set_root_cache_directory(str(self.workdir))\n\n    # Increase the number of retries to avoid spurious server failures\n    self.retry_policy = openml.config.retry_policy\n    self.connection_n_retries = openml.config.connection_n_retries\n    openml.config.set_retry_policy(\"robot\", n_retries=20)\n</code></pre>"},{"location":"reference/testing/#openml.testing.TestBase.tearDown","title":"<code>tearDown()</code>","text":"<p>Tear down the test</p> Source code in <code>openml/testing.py</code> <pre><code>def tearDown(self) -&gt; None:\n    \"\"\"Tear down the test\"\"\"\n    os.chdir(self.cwd)\n    try:\n        shutil.rmtree(self.workdir)\n    except PermissionError as e:\n        if os.name != \"nt\":\n            # one of the files may still be used by another process\n            raise e\n\n    openml.config.server = self.production_server\n    openml.config.connection_n_retries = self.connection_n_retries\n    openml.config.retry_policy = self.retry_policy\n</code></pre>"},{"location":"reference/testing/#openml.testing.check_task_existence","title":"<code>check_task_existence(task_type, dataset_id, target_name, **kwargs)</code>","text":"<p>Checks if any task with exists on test server that matches the meta data.</p> Parameter <p>task_type : openml.tasks.TaskType dataset_id : int target_name : str</p> Return <p>int, None</p> Source code in <code>openml/testing.py</code> <pre><code>def check_task_existence(\n    task_type: TaskType,\n    dataset_id: int,\n    target_name: str,\n    **kwargs: dict[str, str | int | dict[str, str | int | openml.tasks.TaskType]],\n) -&gt; int | None:\n    \"\"\"Checks if any task with exists on test server that matches the meta data.\n\n    Parameter\n    ---------\n    task_type : openml.tasks.TaskType\n    dataset_id : int\n    target_name : str\n\n    Return\n    ------\n    int, None\n    \"\"\"\n    return_val = None\n    tasks = openml.tasks.list_tasks(task_type=task_type, output_format=\"dataframe\")\n    assert isinstance(tasks, pd.DataFrame)\n    if len(tasks) == 0:\n        return None\n    tasks = tasks.loc[tasks[\"did\"] == dataset_id]\n    if len(tasks) == 0:\n        return None\n    tasks = tasks.loc[tasks[\"target_feature\"] == target_name]\n    if len(tasks) == 0:\n        return None\n    task_match = []\n    for task_id in tasks[\"tid\"].to_list():\n        task_match.append(task_id)\n        try:\n            task = openml.tasks.get_task(task_id)\n        except OpenMLServerException:\n            # can fail if task_id deleted by another parallely run unit test\n            task_match.pop(-1)\n            return_val = None\n            continue\n        for k, v in kwargs.items():\n            if getattr(task, k) != v:\n                # even if one of the meta-data key mismatches, then task_id is not a match\n                task_match.pop(-1)\n                break\n        # if task_id is retained in the task_match list, it passed all meta key-value matches\n        if len(task_match) == 1:\n            return_val = task_id\n            break\n    if len(task_match) == 0:\n        return_val = None\n    return return_val\n</code></pre>"},{"location":"reference/utils/","title":"utils","text":""},{"location":"reference/utils/#openml.utils.extract_xml_tags","title":"<code>extract_xml_tags(xml_tag_name, node, *, allow_none=True)</code>","text":"<pre><code>extract_xml_tags(xml_tag_name: str, node: Mapping[str, Any], *, allow_none: Literal[True] = ...) -&gt; Any | None\n</code></pre><pre><code>extract_xml_tags(xml_tag_name: str, node: Mapping[str, Any], *, allow_none: Literal[False]) -&gt; Any\n</code></pre> <p>Helper to extract xml tags from xmltodict.</p> <p>Parameters:</p> Name Type Description Default <code>xml_tag_name</code> <code>str</code> <p>Name of the xml tag to extract from the node.</p> required <code>node</code> <code>Mapping[str, Any]</code> <p>Node object returned by <code>xmltodict</code> from which <code>xml_tag_name</code> should be extracted.</p> required <code>allow_none</code> <code>bool</code> <p>If <code>False</code>, the tag needs to exist in the node. Will raise a <code>ValueError</code> if it does not.</p> <code>True</code> <p>Returns:</p> Type Description <code>object</code> Source code in <code>openml/utils.py</code> <pre><code>def extract_xml_tags(\n    xml_tag_name: str,\n    node: Mapping[str, Any],\n    *,\n    allow_none: bool = True,\n) -&gt; Any | None:\n    \"\"\"Helper to extract xml tags from xmltodict.\n\n    Parameters\n    ----------\n    xml_tag_name : str\n        Name of the xml tag to extract from the node.\n\n    node : Mapping[str, Any]\n        Node object returned by ``xmltodict`` from which ``xml_tag_name``\n        should be extracted.\n\n    allow_none : bool\n        If ``False``, the tag needs to exist in the node. Will raise a\n        ``ValueError`` if it does not.\n\n    Returns\n    -------\n    object\n    \"\"\"\n    if xml_tag_name in node and node[xml_tag_name] is not None:\n        if isinstance(node[xml_tag_name], (dict, str)):\n            return [node[xml_tag_name]]\n        if isinstance(node[xml_tag_name], list):\n            return node[xml_tag_name]\n\n        raise ValueError(\"Received not string and non list as tag item\")\n\n    if allow_none:\n        return None\n\n    raise ValueError(f\"Could not find tag '{xml_tag_name}' in node '{node!s}'\")\n</code></pre>"},{"location":"reference/datasets/","title":"datasets","text":""},{"location":"reference/datasets/#openml.datasets.OpenMLDataFeature","title":"<code>OpenMLDataFeature</code>","text":"<p>Data Feature (a.k.a. Attribute) object.</p> <p>Parameters:</p> Name Type Description Default <code>index</code> <code>int</code> <p>The index of this feature</p> required <code>name</code> <code>str</code> <p>Name of the feature</p> required <code>data_type</code> <code>str</code> <p>can be nominal, numeric, string, date (corresponds to arff)</p> required <code>nominal_values</code> <code>list(str)</code> <p>list of the possible values, in case of nominal attribute</p> required <code>number_missing_values</code> <code>int</code> <p>Number of rows that have a missing value for this feature.</p> required <code>ontologies</code> <code>list(str)</code> <p>list of ontologies attached to this feature. An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided.</p> <code>None</code> Source code in <code>openml/datasets/data_feature.py</code> <pre><code>class OpenMLDataFeature:\n    \"\"\"\n    Data Feature (a.k.a. Attribute) object.\n\n    Parameters\n    ----------\n    index : int\n        The index of this feature\n    name : str\n        Name of the feature\n    data_type : str\n        can be nominal, numeric, string, date (corresponds to arff)\n    nominal_values : list(str)\n        list of the possible values, in case of nominal attribute\n    number_missing_values : int\n        Number of rows that have a missing value for this feature.\n    ontologies : list(str)\n        list of ontologies attached to this feature. An ontology describes the\n        concept that are described in a feature. An ontology is defined by an\n        URL where the information is provided.\n    \"\"\"\n\n    LEGAL_DATA_TYPES: ClassVar[Sequence[str]] = [\"nominal\", \"numeric\", \"string\", \"date\"]\n\n    def __init__(  # noqa: PLR0913\n        self,\n        index: int,\n        name: str,\n        data_type: str,\n        nominal_values: list[str],\n        number_missing_values: int,\n        ontologies: list[str] | None = None,\n    ):\n        if not isinstance(index, int):\n            raise TypeError(f\"Index must be `int` but is {type(index)}\")\n\n        if data_type not in self.LEGAL_DATA_TYPES:\n            raise ValueError(\n                f\"data type should be in {self.LEGAL_DATA_TYPES!s}, found: {data_type}\",\n            )\n\n        if data_type == \"nominal\":\n            if nominal_values is None:\n                raise TypeError(\n                    \"Dataset features require attribute `nominal_values` for nominal \"\n                    \"feature type.\",\n                )\n\n            if not isinstance(nominal_values, list):\n                raise TypeError(\n                    \"Argument `nominal_values` is of wrong datatype, should be list, \"\n                    f\"but is {type(nominal_values)}\",\n                )\n        elif nominal_values is not None:\n            raise TypeError(\"Argument `nominal_values` must be None for non-nominal feature.\")\n\n        if not isinstance(number_missing_values, int):\n            msg = f\"number_missing_values must be int but is {type(number_missing_values)}\"\n            raise TypeError(msg)\n\n        self.index = index\n        self.name = str(name)\n        self.data_type = str(data_type)\n        self.nominal_values = nominal_values\n        self.number_missing_values = number_missing_values\n        self.ontologies = ontologies\n\n    def __repr__(self) -&gt; str:\n        return \"[%d - %s (%s)]\" % (self.index, self.name, self.data_type)\n\n    def __eq__(self, other: Any) -&gt; bool:\n        return isinstance(other, OpenMLDataFeature) and self.__dict__ == other.__dict__\n\n    def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -&gt; None:  # noqa: FBT001, ARG002\n        pp.text(str(self))\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset","title":"<code>OpenMLDataset</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>Dataset object.</p> <p>Allows fetching and uploading datasets to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>data_format</code> <code>str</code> <p>Format of the dataset which can be either 'arff' or 'sparse_arff'.</p> <code>'arff'</code> <code>cache_format</code> <code>str</code> <p>Format for caching the dataset which can be either 'feather' or 'pickle'.</p> <code>'pickle'</code> <code>dataset_id</code> <code>int</code> <p>Id autogenerated by the server.</p> <code>None</code> <code>version</code> <code>int</code> <p>Version of this dataset. '1' for original version. Auto-incremented by server.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>The date-time when the dataset was uploaded, generated by server.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>licence</code> <code>str</code> <p>License of the data.</p> <code>None</code> <code>url</code> <code>str</code> <p>Valid URL, points to actual data file. The file can be on the OpenML server or another dataset repository.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>version_label</code> <code>str</code> <p>Version label provided by user. Can be a date, hash, or some other type of id.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>tag</code> <code>str</code> <p>Tags, describing the algorithms.</p> <code>None</code> <code>visibility</code> <code>str</code> <p>Who can see the dataset. Typical values: 'Everyone','All my friends','Only me'. Can also be any of the user's circles.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <code>md5_checksum</code> <code>str</code> <p>MD5 checksum to check if the dataset is downloaded without corruption.</p> <code>None</code> <code>data_file</code> <code>str</code> <p>Path to where the dataset is located.</p> <code>None</code> <code>features_file</code> <code>dict</code> <p>A dictionary of dataset features, which maps a feature index to a OpenMLDataFeature.</p> <code>None</code> <code>qualities_file</code> <code>dict</code> <p>A dictionary of dataset qualities, which maps a quality name to a quality value.</p> <code>None</code> <code>dataset</code> <code>str | None</code> <p>Serialized arff dataset string.</p> <code>None</code> <code>parquet_url</code> <code>str | None</code> <p>This is the URL to the storage location where the dataset files are hosted. This can be a MinIO bucket URL. If specified, the data will be accessed from this URL when reading the files.</p> <code>None</code> <code>parquet_file</code> <code>str | None</code> <p>Path to the local file.</p> <code>None</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>class OpenMLDataset(OpenMLBase):\n    \"\"\"Dataset object.\n\n    Allows fetching and uploading datasets to OpenML.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    data_format : str\n        Format of the dataset which can be either 'arff' or 'sparse_arff'.\n    cache_format : str\n        Format for caching the dataset which can be either 'feather' or 'pickle'.\n    dataset_id : int, optional\n        Id autogenerated by the server.\n    version : int, optional\n        Version of this dataset. '1' for original version.\n        Auto-incremented by server.\n    creator : str, optional\n        The person who created the dataset.\n    contributor : str, optional\n        People who contributed to the current version of the dataset.\n    collection_date : str, optional\n        The date the data was originally collected, given by the uploader.\n    upload_date : str, optional\n        The date-time when the dataset was uploaded, generated by server.\n    language : str, optional\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str, optional\n        License of the data.\n    url : str, optional\n        Valid URL, points to actual data file.\n        The file can be on the OpenML server or another dataset repository.\n    default_target_attribute : str, optional\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column,\n        if present in the dataset.\n    ignore_attribute : str | list, optional\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    version_label : str, optional\n        Version label provided by user.\n        Can be a date, hash, or some other type of id.\n    citation : str, optional\n        Reference(s) that should be cited when building on this data.\n    tag : str, optional\n        Tags, describing the algorithms.\n    visibility : str, optional\n        Who can see the dataset.\n        Typical values: 'Everyone','All my friends','Only me'.\n        Can also be any of the user's circles.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n    md5_checksum : str, optional\n        MD5 checksum to check if the dataset is downloaded without corruption.\n    data_file : str, optional\n        Path to where the dataset is located.\n    features_file : dict, optional\n        A dictionary of dataset features,\n        which maps a feature index to a OpenMLDataFeature.\n    qualities_file : dict, optional\n        A dictionary of dataset qualities,\n        which maps a quality name to a quality value.\n    dataset: string, optional\n        Serialized arff dataset string.\n    parquet_url: string, optional\n        This is the URL to the storage location where the dataset files are hosted.\n        This can be a MinIO bucket URL. If specified, the data will be accessed\n        from this URL when reading the files.\n    parquet_file: string, optional\n        Path to the local file.\n    \"\"\"\n\n    def __init__(  # noqa: C901, PLR0912, PLR0913, PLR0915\n        self,\n        name: str,\n        description: str | None,\n        data_format: Literal[\"arff\", \"sparse_arff\"] = \"arff\",\n        cache_format: Literal[\"feather\", \"pickle\"] = \"pickle\",\n        dataset_id: int | None = None,\n        version: int | None = None,\n        creator: str | None = None,\n        contributor: str | None = None,\n        collection_date: str | None = None,\n        upload_date: str | None = None,\n        language: str | None = None,\n        licence: str | None = None,\n        url: str | None = None,\n        default_target_attribute: str | None = None,\n        row_id_attribute: str | None = None,\n        ignore_attribute: str | list[str] | None = None,\n        version_label: str | None = None,\n        citation: str | None = None,\n        tag: str | None = None,\n        visibility: str | None = None,\n        original_data_url: str | None = None,\n        paper_url: str | None = None,\n        update_comment: str | None = None,\n        md5_checksum: str | None = None,\n        data_file: str | None = None,\n        features_file: str | None = None,\n        qualities_file: str | None = None,\n        dataset: str | None = None,\n        parquet_url: str | None = None,\n        parquet_file: str | None = None,\n    ):\n        if cache_format not in [\"feather\", \"pickle\"]:\n            raise ValueError(\n                \"cache_format must be one of 'feather' or 'pickle. \"\n                f\"Invalid format specified: {cache_format}\",\n            )\n\n        def find_invalid_characters(string: str, pattern: str) -&gt; str:\n            invalid_chars = set()\n            regex = re.compile(pattern)\n            for char in string:\n                if not regex.match(char):\n                    invalid_chars.add(char)\n            return \",\".join(\n                [f\"'{char}'\" if char != \"'\" else f'\"{char}\"' for char in invalid_chars],\n            )\n\n        if dataset_id is None:\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if description and not re.match(pattern, description):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(description, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in description: {description}\",\n                )\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if citation and not re.match(pattern, citation):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(citation, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in citation: {citation}\",\n                )\n            pattern = \"^[a-zA-Z0-9_\\\\-\\\\.\\\\(\\\\),]+$\"\n            if not re.match(pattern, name):\n                # regex given by server in error message\n                invalid_characters = find_invalid_characters(name, pattern)\n                raise ValueError(f\"Invalid symbols {invalid_characters} in name: {name}\")\n\n        self.ignore_attribute: list[str] | None = None\n        if isinstance(ignore_attribute, str):\n            self.ignore_attribute = [ignore_attribute]\n        elif isinstance(ignore_attribute, list) or ignore_attribute is None:\n            self.ignore_attribute = ignore_attribute\n        else:\n            raise ValueError(\"Wrong data type for ignore_attribute. Should be list.\")\n\n        # TODO add function to check if the name is casual_string128\n        # Attributes received by querying the RESTful API\n        self.dataset_id = int(dataset_id) if dataset_id is not None else None\n        self.name = name\n        self.version = int(version) if version is not None else None\n        self.description = description\n        self.cache_format = cache_format\n        # Has to be called format, otherwise there will be an XML upload error\n        self.format = data_format\n        self.creator = creator\n        self.contributor = contributor\n        self.collection_date = collection_date\n        self.upload_date = upload_date\n        self.language = language\n        self.licence = licence\n        self.url = url\n        self.default_target_attribute = default_target_attribute\n        self.row_id_attribute = row_id_attribute\n\n        self.version_label = version_label\n        self.citation = citation\n        self.tag = tag\n        self.visibility = visibility\n        self.original_data_url = original_data_url\n        self.paper_url = paper_url\n        self.update_comment = update_comment\n        self.md5_checksum = md5_checksum\n        self.data_file = data_file\n        self.parquet_file = parquet_file\n        self._dataset = dataset\n        self._parquet_url = parquet_url\n\n        self._features: dict[int, OpenMLDataFeature] | None = None\n        self._qualities: dict[str, float] | None = None\n        self._no_qualities_found = False\n\n        if features_file is not None:\n            self._features = _read_features(Path(features_file))\n\n        # \"\" was the old default value by `get_dataset` and maybe still used by some\n        if qualities_file == \"\":\n            # TODO(0.15): to switch to \"qualities_file is not None\" below and remove warning\n            warnings.warn(\n                \"Starting from Version 0.15 `qualities_file` must be None and not an empty string \"\n                \"to avoid reading the qualities from file. Set `qualities_file` to None to avoid \"\n                \"this warning.\",\n                FutureWarning,\n                stacklevel=2,\n            )\n            qualities_file = None\n\n        if qualities_file is not None:\n            self._qualities = _read_qualities(Path(qualities_file))\n\n        if data_file is not None:\n            data_pickle, data_feather, feather_attribute = self._compressed_cache_file_paths(\n                Path(data_file)\n            )\n            self.data_pickle_file = data_pickle if Path(data_pickle).exists() else None\n            self.data_feather_file = data_feather if Path(data_feather).exists() else None\n            self.feather_attribute_file = feather_attribute if Path(feather_attribute) else None\n        else:\n            self.data_pickle_file = None\n            self.data_feather_file = None\n            self.feather_attribute_file = None\n\n    @property\n    def features(self) -&gt; dict[int, OpenMLDataFeature]:\n        \"\"\"Get the features of this dataset.\"\"\"\n        if self._features is None:\n            # TODO(eddiebergman): These should return a value so we can set it to be not None\n            self._load_features()\n\n        assert self._features is not None\n        return self._features\n\n    @property\n    def qualities(self) -&gt; dict[str, float] | None:\n        \"\"\"Get the qualities of this dataset.\"\"\"\n        # TODO(eddiebergman): Better docstring, I don't know what qualities means\n\n        # We have to check `_no_qualities_found` as there might not be qualities for a dataset\n        if self._qualities is None and (not self._no_qualities_found):\n            self._load_qualities()\n\n        return self._qualities\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Get the dataset numeric id.\"\"\"\n        return self.dataset_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Obtain number of features in accordance with lazy loading.\n        n_features: int | None = None\n        if self._qualities is not None and self._qualities[\"NumberOfFeatures\"] is not None:\n            n_features = int(self._qualities[\"NumberOfFeatures\"])\n        elif self._features is not None:\n            n_features = len(self._features)\n\n        fields: dict[str, int | str | None] = {\n            \"Name\": self.name,\n            \"Version\": self.version,\n            \"Format\": self.format,\n            \"Licence\": self.licence,\n            \"Download URL\": self.url,\n            \"Data file\": str(self.data_file) if self.data_file is not None else None,\n            \"Pickle file\": (\n                str(self.data_pickle_file) if self.data_pickle_file is not None else None\n            ),\n            \"# of features\": n_features,\n        }\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.dataset_id is not None:\n            fields[\"OpenML URL\"] = self.openml_url\n        if self._qualities is not None and self._qualities[\"NumberOfInstances\"] is not None:\n            fields[\"# of instances\"] = int(self._qualities[\"NumberOfInstances\"])\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Name\",\n            \"Version\",\n            \"Format\",\n            \"Upload Date\",\n            \"Licence\",\n            \"Download URL\",\n            \"OpenML URL\",\n            \"Data File\",\n            \"Pickle File\",\n            \"# of features\",\n            \"# of instances\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if not isinstance(other, OpenMLDataset):\n            return False\n\n        server_fields = {\n            \"dataset_id\",\n            \"version\",\n            \"upload_date\",\n            \"url\",\n            \"dataset\",\n            \"data_file\",\n        }\n\n        # check that common keys and values are identical\n        self_keys = set(self.__dict__.keys()) - server_fields\n        other_keys = set(other.__dict__.keys()) - server_fields\n        return self_keys == other_keys and all(\n            self.__dict__[key] == other.__dict__[key] for key in self_keys\n        )\n\n    def _download_data(self) -&gt; None:\n        \"\"\"Download ARFF data file to standard cache directory. Set `self.data_file`.\"\"\"\n        # import required here to avoid circular import.\n        from .functions import _get_dataset_arff, _get_dataset_parquet\n\n        self.data_file = str(_get_dataset_arff(self))\n        if self._parquet_url is not None:\n            self.parquet_file = str(_get_dataset_parquet(self))\n\n    def _get_arff(self, format: str) -&gt; dict:  # noqa: A002\n        \"\"\"Read ARFF file and return decoded arff.\n\n        Reads the file referenced in self.data_file.\n\n        Parameters\n        ----------\n        format : str\n            Format of the ARFF file.\n            Must be one of 'arff' or 'sparse_arff' or a string that will be either of those\n            when converted to lower case.\n\n\n\n        Returns\n        -------\n        dict\n            Decoded arff.\n\n        \"\"\"\n        # TODO: add a partial read method which only returns the attribute\n        # headers of the corresponding .arff file!\n        import struct\n\n        filename = self.data_file\n        assert filename is not None\n        filepath = Path(filename)\n\n        bits = 8 * struct.calcsize(\"P\")\n\n        # Files can be considered too large on a 32-bit system,\n        # if it exceeds 120mb (slightly more than covtype dataset size)\n        # This number is somewhat arbitrary.\n        if bits != 64:\n            MB_120 = 120_000_000\n            file_size = filepath.stat().st_size\n            if file_size &gt; MB_120:\n                raise NotImplementedError(\n                    f\"File {filename} too big for {file_size}-bit system ({bits} bytes).\",\n                )\n\n        if format.lower() == \"arff\":\n            return_type = arff.DENSE\n        elif format.lower() == \"sparse_arff\":\n            return_type = arff.COO\n        else:\n            raise ValueError(f\"Unknown data format {format}\")\n\n        def decode_arff(fh: Any) -&gt; dict:\n            decoder = arff.ArffDecoder()\n            return decoder.decode(fh, encode_nominal=True, return_type=return_type)  # type: ignore\n\n        if filepath.suffix.endswith(\".gz\"):\n            with gzip.open(filename) as zipfile:\n                return decode_arff(zipfile)\n        else:\n            with filepath.open(encoding=\"utf8\") as fh:\n                return decode_arff(fh)\n\n    def _parse_data_from_arff(  # noqa: C901, PLR0912, PLR0915\n        self,\n        arff_file_path: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Parse all required data from arff file.\n\n        Parameters\n        ----------\n        arff_file_path : str\n            Path to the file on disk.\n\n        Returns\n        -------\n        Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]\n            DataFrame or csr_matrix: dataset\n            List[bool]: List indicating which columns contain categorical variables.\n            List[str]: List of column names.\n        \"\"\"\n        try:\n            data = self._get_arff(self.format)\n        except OSError as e:\n            logger.critical(\n                f\"Please check that the data file {arff_file_path} is \" \"there and can be read.\",\n            )\n            raise e\n\n        ARFF_DTYPES_TO_PD_DTYPE = {\n            \"INTEGER\": \"integer\",\n            \"REAL\": \"floating\",\n            \"NUMERIC\": \"floating\",\n            \"STRING\": \"string\",\n        }\n        attribute_dtype = {}\n        attribute_names = []\n        categories_names = {}\n        categorical = []\n        for name, type_ in data[\"attributes\"]:\n            # if the feature is nominal and a sparse matrix is\n            # requested, the categories need to be numeric\n            if isinstance(type_, list) and self.format.lower() == \"sparse_arff\":\n                try:\n                    # checks if the strings which should be the class labels\n                    # can be encoded into integers\n                    pd.factorize(type_)[0]\n                except ValueError as e:\n                    raise ValueError(\n                        \"Categorical data needs to be numeric when using sparse ARFF.\"\n                    ) from e\n\n            # string can only be supported with pandas DataFrame\n            elif type_ == \"STRING\" and self.format.lower() == \"sparse_arff\":\n                raise ValueError(\"Dataset containing strings is not supported with sparse ARFF.\")\n\n            # infer the dtype from the ARFF header\n            if isinstance(type_, list):\n                categorical.append(True)\n                categories_names[name] = type_\n                if len(type_) == 2:\n                    type_norm = [cat.lower().capitalize() for cat in type_]\n                    if {\"True\", \"False\"} == set(type_norm):\n                        categories_names[name] = [cat == \"True\" for cat in type_norm]\n                        attribute_dtype[name] = \"boolean\"\n                    else:\n                        attribute_dtype[name] = \"categorical\"\n                else:\n                    attribute_dtype[name] = \"categorical\"\n            else:\n                categorical.append(False)\n                attribute_dtype[name] = ARFF_DTYPES_TO_PD_DTYPE[type_]\n            attribute_names.append(name)\n\n        if self.format.lower() == \"sparse_arff\":\n            X = data[\"data\"]\n            X_shape = (max(X[1]) + 1, max(X[2]) + 1)\n            X = scipy.sparse.coo_matrix((X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32)\n            X = X.tocsr()\n        elif self.format.lower() == \"arff\":\n            X = pd.DataFrame(data[\"data\"], columns=attribute_names)\n\n            col = []\n            for column_name in X.columns:\n                if attribute_dtype[column_name] in (\"categorical\", \"boolean\"):\n                    categories = self._unpack_categories(\n                        X[column_name],  # type: ignore\n                        categories_names[column_name],\n                    )\n                    col.append(categories)\n                elif attribute_dtype[column_name] in (\"floating\", \"integer\"):\n                    X_col = X[column_name]\n                    if X_col.min() &gt;= 0 and X_col.max() &lt;= 255:\n                        try:\n                            X_col_uint = X_col.astype(\"uint8\")\n                            if (X_col == X_col_uint).all():\n                                col.append(X_col_uint)\n                                continue\n                        except ValueError:\n                            pass\n                    col.append(X[column_name])\n                else:\n                    col.append(X[column_name])\n            X = pd.concat(col, axis=1)\n        else:\n            raise ValueError(f\"Dataset format '{self.format}' is not a valid format.\")\n\n        return X, categorical, attribute_names  # type: ignore\n\n    def _compressed_cache_file_paths(self, data_file: Path) -&gt; tuple[Path, Path, Path]:\n        data_pickle_file = data_file.with_suffix(\".pkl.py3\")\n        data_feather_file = data_file.with_suffix(\".feather\")\n        feather_attribute_file = data_file.with_suffix(\".feather.attributes.pkl.py3\")\n        return data_pickle_file, data_feather_file, feather_attribute_file\n\n    def _cache_compressed_file_from_file(\n        self,\n        data_file: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Store data from the local file in compressed format.\n\n        If a local parquet file is present it will be used instead of the arff file.\n        Sets cache_format to 'pickle' if data is sparse.\n        \"\"\"\n        (\n            data_pickle_file,\n            data_feather_file,\n            feather_attribute_file,\n        ) = self._compressed_cache_file_paths(data_file)\n\n        if data_file.suffix == \".arff\":\n            data, categorical, attribute_names = self._parse_data_from_arff(data_file)\n        elif data_file.suffix == \".pq\":\n            try:\n                data = pd.read_parquet(data_file)\n            except Exception as e:  # noqa: BLE001\n                raise Exception(f\"File: {data_file}\") from e\n\n            categorical = [data[c].dtype.name == \"category\" for c in data.columns]\n            attribute_names = list(data.columns)\n        else:\n            raise ValueError(f\"Unknown file type for file '{data_file}'.\")\n\n        # Feather format does not work for sparse datasets, so we use pickle for sparse datasets\n        if scipy.sparse.issparse(data):\n            self.cache_format = \"pickle\"\n\n        logger.info(f\"{self.cache_format} write {self.name}\")\n        if self.cache_format == \"feather\":\n            assert isinstance(data, pd.DataFrame)\n\n            data.to_feather(data_feather_file)\n            with open(feather_attribute_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_feather_file = data_feather_file\n            self.feather_attribute_file = feather_attribute_file\n\n        else:\n            with open(data_pickle_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_pickle_file = data_pickle_file\n\n        data_file = data_pickle_file if self.cache_format == \"pickle\" else data_feather_file\n        logger.debug(f\"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}\")\n\n        return data, categorical, attribute_names\n\n    def _load_data(self) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:  # noqa: PLR0912, C901\n        \"\"\"Load data from compressed format or arff. Download data if not present on disk.\"\"\"\n        need_to_create_pickle = self.cache_format == \"pickle\" and self.data_pickle_file is None\n        need_to_create_feather = self.cache_format == \"feather\" and self.data_feather_file is None\n\n        if need_to_create_pickle or need_to_create_feather:\n            if self.data_file is None:\n                self._download_data()\n\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n\n        # helper variable to help identify where errors occur\n        fpath = self.data_feather_file if self.cache_format == \"feather\" else self.data_pickle_file\n        logger.info(f\"{self.cache_format} load data {self.name}\")\n        try:\n            if self.cache_format == \"feather\":\n                assert self.data_feather_file is not None\n                assert self.feather_attribute_file is not None\n\n                data = pd.read_feather(self.data_feather_file)\n                fpath = self.feather_attribute_file\n                with open(self.feather_attribute_file, \"rb\") as fh:  # noqa: PTH123\n                    categorical, attribute_names = pickle.load(fh)  # noqa: S301\n            else:\n                assert self.data_pickle_file is not None\n                with open(self.data_pickle_file, \"rb\") as fh:  # noqa: PTH123\n                    data, categorical, attribute_names = pickle.load(fh)  # noqa: S301\n        except FileNotFoundError as e:\n            raise ValueError(\n                f\"Cannot find file for dataset {self.name} at location '{fpath}'.\"\n            ) from e\n        except (EOFError, ModuleNotFoundError, ValueError, AttributeError) as e:\n            error_message = getattr(e, \"message\", e.args[0])\n            hint = \"\"\n\n            if isinstance(e, EOFError):\n                readable_error = \"Detected a corrupt cache file\"\n            elif isinstance(e, (ModuleNotFoundError, AttributeError)):\n                readable_error = \"Detected likely dependency issues\"\n                hint = (\n                    \"This can happen if the cache was constructed with a different pandas version \"\n                    \"than the one that is used to load the data. See also \"\n                )\n                if isinstance(e, ModuleNotFoundError):\n                    hint += \"https://github.com/openml/openml-python/issues/918. \"\n                elif isinstance(e, AttributeError):\n                    hint += \"https://github.com/openml/openml-python/pull/1121. \"\n\n            elif isinstance(e, ValueError) and \"unsupported pickle protocol\" in e.args[0]:\n                readable_error = \"Encountered unsupported pickle protocol\"\n            else:\n                raise e\n\n            logger.warning(\n                f\"{readable_error} when loading dataset {self.id} from '{fpath}'. \"\n                f\"{hint}\"\n                f\"Error message was: {error_message}. \"\n                \"We will continue loading data from the arff-file, \"\n                \"but this will be much slower for big datasets. \"\n                \"Please manually delete the cache file if you want OpenML-Python \"\n                \"to attempt to reconstruct it.\",\n            )\n            assert self.data_file is not None\n            data, categorical, attribute_names = self._parse_data_from_arff(Path(self.data_file))\n\n        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)\n        if self.cache_format == \"pickle\" and not data_up_to_date:\n            logger.info(\"Updating outdated pickle file.\")\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n        return data, categorical, attribute_names\n\n    # TODO(eddiebergman): Can type this better with overload\n    # TODO(eddiebergman): Could also techinically use scipy.sparse.sparray\n    @staticmethod\n    def _convert_array_format(\n        data: pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix,\n        array_format: Literal[\"array\", \"dataframe\"],\n        attribute_names: list | None = None,\n    ) -&gt; pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix:\n        \"\"\"Convert a dataset to a given array format.\n\n        Converts to numpy array if data is non-sparse.\n        Converts to a sparse dataframe if data is sparse.\n\n        Parameters\n        ----------\n        array_format : str {'array', 'dataframe'}\n            Desired data type of the output\n            - If array_format='array'\n                If data is non-sparse\n                    Converts to numpy-array\n                    Enforces numeric encoding of categorical columns\n                    Missing values are represented as NaN in the numpy-array\n                else returns data as is\n            - If array_format='dataframe'\n                If data is sparse\n                    Works only on sparse data\n                    Converts sparse data to sparse dataframe\n                else returns data as is\n\n        \"\"\"\n        if array_format == \"array\" and not isinstance(data, scipy.sparse.spmatrix):\n            # We encode the categories such that they are integer to be able\n            # to make a conversion to numeric for backward compatibility\n            def _encode_if_category(column: pd.Series | np.ndarray) -&gt; pd.Series | np.ndarray:\n                if column.dtype.name == \"category\":\n                    column = column.cat.codes.astype(np.float32)\n                    mask_nan = column == -1\n                    column[mask_nan] = np.nan\n                return column\n\n            if isinstance(data, pd.DataFrame):\n                columns = {\n                    column_name: _encode_if_category(data.loc[:, column_name])\n                    for column_name in data.columns\n                }\n                data = pd.DataFrame(columns)\n            else:\n                data = _encode_if_category(data)\n\n            try:\n                # TODO(eddiebergman): float32?\n                return_array = np.asarray(data, dtype=np.float32)\n            except ValueError as e:\n                raise PyOpenMLError(\n                    \"PyOpenML cannot handle string when returning numpy\"\n                    ' arrays. Use dataset_format=\"dataframe\".',\n                ) from e\n\n            return return_array\n\n        if array_format == \"dataframe\":\n            if scipy.sparse.issparse(data):\n                data = pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)\n        else:\n            data_type = \"sparse-data\" if scipy.sparse.issparse(data) else \"non-sparse data\"\n            logger.warning(\n                f\"Cannot convert {data_type} ({type(data)}) to '{array_format}'.\"\n                \" Returning input data.\",\n            )\n        return data\n\n    @staticmethod\n    def _unpack_categories(series: pd.Series, categories: list) -&gt; pd.Series:\n        # nan-likes can not be explicitly specified as a category\n        def valid_category(cat: Any) -&gt; bool:\n            return isinstance(cat, str) or (cat is not None and not np.isnan(cat))\n\n        filtered_categories = [c for c in categories if valid_category(c)]\n        col = []\n        for x in series:\n            try:\n                col.append(categories[int(x)])\n            except (TypeError, ValueError):\n                col.append(np.nan)\n\n        # We require two lines to create a series of categories as detailed here:\n        # https://pandas.pydata.org/pandas-docs/version/0.24/user_guide/categorical.html#series-creation\n        raw_cat = pd.Categorical(col, ordered=True, categories=filtered_categories)\n        return pd.Series(raw_cat, index=series.index, name=series.name)\n\n    def get_data(  # noqa: C901, PLR0912, PLR0915\n        self,\n        target: list[str] | str | None = None,\n        include_row_id: bool = False,  # noqa: FBT001, FBT002\n        include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n        np.ndarray | pd.DataFrame | None,\n        list[bool],\n        list[str],\n    ]:\n        \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n        Parameters\n        ----------\n        target : string, List[str] or None (default=None)\n            Name of target column to separate from the data.\n            Splitting multiple columns is currently not supported.\n        include_row_id : boolean (default=False)\n            Whether to include row ids in the returned dataset.\n        include_ignore_attribute : boolean (default=False)\n            Whether to include columns that are marked as \"ignore\"\n            on the server in the dataset.\n        dataset_format : string (default='dataframe')\n            The format of returned dataset.\n            If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n            matrix. Support for ``array`` will be removed in 0.15.\n            If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n        Returns\n        -------\n        X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n            Dataset\n        y : ndarray or pd.Series, shape (n_samples, ) or None\n            Target column\n        categorical_indicator : boolean ndarray\n            Mask that indicate categorical features.\n        attribute_names : List[str]\n            List of attribute names.\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        data, categorical, attribute_names = self._load_data()\n\n        to_exclude = []\n        if not include_row_id and self.row_id_attribute is not None:\n            if isinstance(self.row_id_attribute, str):\n                to_exclude.append(self.row_id_attribute)\n            elif isinstance(self.row_id_attribute, Iterable):\n                to_exclude.extend(self.row_id_attribute)\n\n        if not include_ignore_attribute and self.ignore_attribute is not None:\n            if isinstance(self.ignore_attribute, str):\n                to_exclude.append(self.ignore_attribute)\n            elif isinstance(self.ignore_attribute, Iterable):\n                to_exclude.extend(self.ignore_attribute)\n\n        if len(to_exclude) &gt; 0:\n            logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n            keep = np.array([column not in to_exclude for column in attribute_names])\n            data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n            categorical = [cat for cat, k in zip(categorical, keep) if k]\n            attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n        if target is None:\n            data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n            targets = None\n        else:\n            if isinstance(target, str):\n                target = target.split(\",\") if \",\" in target else [target]\n            targets = np.array([column in target for column in attribute_names])\n            target_names = [column for column in attribute_names if column in target]\n            if np.sum(targets) &gt; 1:\n                raise NotImplementedError(\n                    \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n                )\n            target_categorical = [\n                cat for cat, column in zip(categorical, attribute_names) if column in target\n            ]\n            target_dtype = int if target_categorical[0] else float\n\n            if isinstance(data, pd.DataFrame):\n                x = data.iloc[:, ~targets]\n                y = data.iloc[:, targets]\n            else:\n                x = data[:, ~targets]\n                y = data[:, targets].astype(target_dtype)  # type: ignore\n\n            categorical = [cat for cat, t in zip(categorical, targets) if not t]\n            attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n            x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n            if dataset_format == \"array\" and scipy.sparse.issparse(y):\n                # scikit-learn requires dense representation of targets\n                y = np.asarray(y.todense()).astype(target_dtype)\n                # dense representation of single column sparse arrays become a 2-d array\n                # need to flatten it to a 1-d array for _convert_array_format()\n                y = y.squeeze()\n            y = self._convert_array_format(y, dataset_format, target_names)\n            y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n            if len(y.shape) &gt; 1 and y.shape[1] == 1:\n                # single column targets should be 1-d for both `array` and `dataframe` formats\n                assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n                y = y.squeeze()\n            data, targets = x, y\n\n        return data, targets, categorical, attribute_names  # type: ignore\n\n    def _load_features(self) -&gt; None:\n        \"\"\"Load the features metadata from the server and store it in the dataset object.\"\"\"\n        # Delayed Import to avoid circular imports or having to import all of dataset.functions to\n        # import OpenMLDataset.\n        from openml.datasets.functions import _get_dataset_features_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        features_file = _get_dataset_features_file(None, self.dataset_id)\n        self._features = _read_features(features_file)\n\n    def _load_qualities(self) -&gt; None:\n        \"\"\"Load qualities information from the server and store it in the dataset object.\"\"\"\n        # same reason as above for _load_features\n        from openml.datasets.functions import _get_dataset_qualities_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        qualities_file = _get_dataset_qualities_file(None, self.dataset_id)\n\n        if qualities_file is None:\n            self._no_qualities_found = True\n        else:\n            self._qualities = _read_qualities(qualities_file)\n\n    def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n        \"\"\"Reads the datasets arff to determine the class-labels.\n\n        If the task has no class labels (for example a regression problem)\n        it returns None. Necessary because the data returned by get_data\n        only contains the indices of the classes, while OpenML needs the real\n        classname when uploading the results of a run.\n\n        Parameters\n        ----------\n        target_name : str\n            Name of the target attribute\n\n        Returns\n        -------\n        list\n        \"\"\"\n        for feature in self.features.values():\n            if feature.name == target_name:\n                if feature.data_type == \"nominal\":\n                    return feature.nominal_values\n\n                if feature.data_type == \"string\":\n                    # Rel.: #1311\n                    # The target is invalid for a classification task if the feature type is string\n                    # and not nominal. For such miss-configured tasks, we silently fix it here as\n                    # we can safely interpreter string as nominal.\n                    df, *_ = self.get_data()\n                    return list(df[feature.name].unique())\n\n        return None\n\n    def get_features_by_type(  # noqa: C901\n        self,\n        data_type: str,\n        exclude: list[str] | None = None,\n        exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n        exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; list[int]:\n        \"\"\"\n        Return indices of features of a given type, e.g. all nominal features.\n        Optional parameters to exclude various features by index or ontology.\n\n        Parameters\n        ----------\n        data_type : str\n            The data type to return (e.g., nominal, numeric, date, string)\n        exclude : list(int)\n            List of columns to exclude from the return value\n        exclude_ignore_attribute : bool\n            Whether to exclude the defined ignore attributes (and adapt the\n            return values as if these indices are not present)\n        exclude_row_id_attribute : bool\n            Whether to exclude the defined row id attributes (and adapt the\n            return values as if these indices are not present)\n\n        Returns\n        -------\n        result : list\n            a list of indices that have the specified data type\n        \"\"\"\n        if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n            raise TypeError(\"Illegal feature type requested\")\n        if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n            raise TypeError(\"ignore_attribute should be a list\")\n        if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n            raise TypeError(\"row id attribute should be a str\")\n        if exclude is not None and not isinstance(exclude, list):\n            raise TypeError(\"Exclude should be a list\")\n            # assert all(isinstance(elem, str) for elem in exclude),\n            #            \"Exclude should be a list of strings\"\n        to_exclude = []\n        if exclude is not None:\n            to_exclude.extend(exclude)\n        if exclude_ignore_attribute and self.ignore_attribute is not None:\n            to_exclude.extend(self.ignore_attribute)\n        if exclude_row_id_attribute and self.row_id_attribute is not None:\n            to_exclude.append(self.row_id_attribute)\n\n        result = []\n        offset = 0\n        # this function assumes that everything in to_exclude will\n        # be 'excluded' from the dataset (hence the offset)\n        for idx in self.features:\n            name = self.features[idx].name\n            if name in to_exclude:\n                offset += 1\n            elif self.features[idx].data_type == data_type:\n                result.append(idx - offset)\n        return result\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Adds the 'dataset' to file elements.\"\"\"\n        file_elements: dict = {}\n        path = None if self.data_file is None else Path(self.data_file).absolute()\n\n        if self._dataset is not None:\n            file_elements[\"dataset\"] = self._dataset\n        elif path is not None and path.exists():\n            with path.open(\"rb\") as fp:\n                file_elements[\"dataset\"] = fp.read()\n\n            try:\n                dataset_utf8 = str(file_elements[\"dataset\"], encoding=\"utf8\")\n                arff.ArffDecoder().decode(dataset_utf8, encode_nominal=True)\n            except arff.ArffException as e:\n                raise ValueError(\"The file you have provided is not a valid arff file.\") from e\n\n        elif self.url is None:\n            raise ValueError(\"No valid url/path to the data file was given.\")\n        return file_elements\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.dataset_id = int(xml_response[\"oml:upload_data_set\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        props = [\n            \"id\",\n            \"name\",\n            \"version\",\n            \"description\",\n            \"format\",\n            \"creator\",\n            \"contributor\",\n            \"collection_date\",\n            \"upload_date\",\n            \"language\",\n            \"licence\",\n            \"url\",\n            \"default_target_attribute\",\n            \"row_id_attribute\",\n            \"ignore_attribute\",\n            \"version_label\",\n            \"citation\",\n            \"tag\",\n            \"visibility\",\n            \"original_data_url\",\n            \"paper_url\",\n            \"update_comment\",\n            \"md5_checksum\",\n        ]\n\n        prop_values = {}\n        for prop in props:\n            content = getattr(self, prop, None)\n            if content is not None:\n                prop_values[\"oml:\" + prop] = content\n\n        return {\n            \"oml:data_set_description\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **prop_values,\n            }\n        }\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.features","title":"<code>features: dict[int, OpenMLDataFeature]</code>  <code>property</code>","text":"<p>Get the features of this dataset.</p>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Get the dataset numeric id.</p>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.qualities","title":"<code>qualities: dict[str, float] | None</code>  <code>property</code>","text":"<p>Get the qualities of this dataset.</p>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.get_data","title":"<code>get_data(target=None, include_row_id=False, include_ignore_attribute=False, dataset_format='dataframe')</code>","text":"<p>Returns dataset content as dataframes or sparse matrices.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>(string, List[str] or None(default=None))</code> <p>Name of target column to separate from the data. Splitting multiple columns is currently not supported.</p> <code>None</code> <code>include_row_id</code> <code>boolean(default=False)</code> <p>Whether to include row ids in the returned dataset.</p> <code>False</code> <code>include_ignore_attribute</code> <code>boolean(default=False)</code> <p>Whether to include columns that are marked as \"ignore\" on the server in the dataset.</p> <code>False</code> <code>dataset_format</code> <code>string(default='dataframe')</code> <p>The format of returned dataset. If <code>array</code>, the returned dataset will be a NumPy array or a SciPy sparse matrix. Support for <code>array</code> will be removed in 0.15. If <code>dataframe</code>, the returned dataset will be a Pandas DataFrame.</p> <code>'dataframe'</code> <p>Returns:</p> Name Type Description <code>X</code> <code>ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)</code> <p>Dataset</p> <code>y</code> <code>(ndarray or Series, shape(n_samples) or None)</code> <p>Target column</p> <code>categorical_indicator</code> <code>boolean ndarray</code> <p>Mask that indicate categorical features.</p> <code>attribute_names</code> <code>List[str]</code> <p>List of attribute names.</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_data(  # noqa: C901, PLR0912, PLR0915\n    self,\n    target: list[str] | str | None = None,\n    include_row_id: bool = False,  # noqa: FBT001, FBT002\n    include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n    np.ndarray | pd.DataFrame | None,\n    list[bool],\n    list[str],\n]:\n    \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n    Parameters\n    ----------\n    target : string, List[str] or None (default=None)\n        Name of target column to separate from the data.\n        Splitting multiple columns is currently not supported.\n    include_row_id : boolean (default=False)\n        Whether to include row ids in the returned dataset.\n    include_ignore_attribute : boolean (default=False)\n        Whether to include columns that are marked as \"ignore\"\n        on the server in the dataset.\n    dataset_format : string (default='dataframe')\n        The format of returned dataset.\n        If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n        matrix. Support for ``array`` will be removed in 0.15.\n        If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n    Returns\n    -------\n    X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n        Dataset\n    y : ndarray or pd.Series, shape (n_samples, ) or None\n        Target column\n    categorical_indicator : boolean ndarray\n        Mask that indicate categorical features.\n    attribute_names : List[str]\n        List of attribute names.\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    data, categorical, attribute_names = self._load_data()\n\n    to_exclude = []\n    if not include_row_id and self.row_id_attribute is not None:\n        if isinstance(self.row_id_attribute, str):\n            to_exclude.append(self.row_id_attribute)\n        elif isinstance(self.row_id_attribute, Iterable):\n            to_exclude.extend(self.row_id_attribute)\n\n    if not include_ignore_attribute and self.ignore_attribute is not None:\n        if isinstance(self.ignore_attribute, str):\n            to_exclude.append(self.ignore_attribute)\n        elif isinstance(self.ignore_attribute, Iterable):\n            to_exclude.extend(self.ignore_attribute)\n\n    if len(to_exclude) &gt; 0:\n        logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n        keep = np.array([column not in to_exclude for column in attribute_names])\n        data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n        categorical = [cat for cat, k in zip(categorical, keep) if k]\n        attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n    if target is None:\n        data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n        targets = None\n    else:\n        if isinstance(target, str):\n            target = target.split(\",\") if \",\" in target else [target]\n        targets = np.array([column in target for column in attribute_names])\n        target_names = [column for column in attribute_names if column in target]\n        if np.sum(targets) &gt; 1:\n            raise NotImplementedError(\n                \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n            )\n        target_categorical = [\n            cat for cat, column in zip(categorical, attribute_names) if column in target\n        ]\n        target_dtype = int if target_categorical[0] else float\n\n        if isinstance(data, pd.DataFrame):\n            x = data.iloc[:, ~targets]\n            y = data.iloc[:, targets]\n        else:\n            x = data[:, ~targets]\n            y = data[:, targets].astype(target_dtype)  # type: ignore\n\n        categorical = [cat for cat, t in zip(categorical, targets) if not t]\n        attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n        x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n        if dataset_format == \"array\" and scipy.sparse.issparse(y):\n            # scikit-learn requires dense representation of targets\n            y = np.asarray(y.todense()).astype(target_dtype)\n            # dense representation of single column sparse arrays become a 2-d array\n            # need to flatten it to a 1-d array for _convert_array_format()\n            y = y.squeeze()\n        y = self._convert_array_format(y, dataset_format, target_names)\n        y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n        if len(y.shape) &gt; 1 and y.shape[1] == 1:\n            # single column targets should be 1-d for both `array` and `dataframe` formats\n            assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n            y = y.squeeze()\n        data, targets = x, y\n\n    return data, targets, categorical, attribute_names  # type: ignore\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.get_features_by_type","title":"<code>get_features_by_type(data_type, exclude=None, exclude_ignore_attribute=True, exclude_row_id_attribute=True)</code>","text":"<p>Return indices of features of a given type, e.g. all nominal features. Optional parameters to exclude various features by index or ontology.</p> <p>Parameters:</p> Name Type Description Default <code>data_type</code> <code>str</code> <p>The data type to return (e.g., nominal, numeric, date, string)</p> required <code>exclude</code> <code>list(int)</code> <p>List of columns to exclude from the return value</p> <code>None</code> <code>exclude_ignore_attribute</code> <code>bool</code> <p>Whether to exclude the defined ignore attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <code>exclude_row_id_attribute</code> <code>bool</code> <p>Whether to exclude the defined row id attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <p>Returns:</p> Name Type Description <code>result</code> <code>list</code> <p>a list of indices that have the specified data type</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_features_by_type(  # noqa: C901\n    self,\n    data_type: str,\n    exclude: list[str] | None = None,\n    exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n    exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n) -&gt; list[int]:\n    \"\"\"\n    Return indices of features of a given type, e.g. all nominal features.\n    Optional parameters to exclude various features by index or ontology.\n\n    Parameters\n    ----------\n    data_type : str\n        The data type to return (e.g., nominal, numeric, date, string)\n    exclude : list(int)\n        List of columns to exclude from the return value\n    exclude_ignore_attribute : bool\n        Whether to exclude the defined ignore attributes (and adapt the\n        return values as if these indices are not present)\n    exclude_row_id_attribute : bool\n        Whether to exclude the defined row id attributes (and adapt the\n        return values as if these indices are not present)\n\n    Returns\n    -------\n    result : list\n        a list of indices that have the specified data type\n    \"\"\"\n    if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n        raise TypeError(\"Illegal feature type requested\")\n    if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n        raise TypeError(\"ignore_attribute should be a list\")\n    if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n        raise TypeError(\"row id attribute should be a str\")\n    if exclude is not None and not isinstance(exclude, list):\n        raise TypeError(\"Exclude should be a list\")\n        # assert all(isinstance(elem, str) for elem in exclude),\n        #            \"Exclude should be a list of strings\"\n    to_exclude = []\n    if exclude is not None:\n        to_exclude.extend(exclude)\n    if exclude_ignore_attribute and self.ignore_attribute is not None:\n        to_exclude.extend(self.ignore_attribute)\n    if exclude_row_id_attribute and self.row_id_attribute is not None:\n        to_exclude.append(self.row_id_attribute)\n\n    result = []\n    offset = 0\n    # this function assumes that everything in to_exclude will\n    # be 'excluded' from the dataset (hence the offset)\n    for idx in self.features:\n        name = self.features[idx].name\n        if name in to_exclude:\n            offset += 1\n        elif self.features[idx].data_type == data_type:\n            result.append(idx - offset)\n    return result\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.OpenMLDataset.retrieve_class_labels","title":"<code>retrieve_class_labels(target_name='class')</code>","text":"<p>Reads the datasets arff to determine the class-labels.</p> <p>If the task has no class labels (for example a regression problem) it returns None. Necessary because the data returned by get_data only contains the indices of the classes, while OpenML needs the real classname when uploading the results of a run.</p> <p>Parameters:</p> Name Type Description Default <code>target_name</code> <code>str</code> <p>Name of the target attribute</p> <code>'class'</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n    \"\"\"Reads the datasets arff to determine the class-labels.\n\n    If the task has no class labels (for example a regression problem)\n    it returns None. Necessary because the data returned by get_data\n    only contains the indices of the classes, while OpenML needs the real\n    classname when uploading the results of a run.\n\n    Parameters\n    ----------\n    target_name : str\n        Name of the target attribute\n\n    Returns\n    -------\n    list\n    \"\"\"\n    for feature in self.features.values():\n        if feature.name == target_name:\n            if feature.data_type == \"nominal\":\n                return feature.nominal_values\n\n            if feature.data_type == \"string\":\n                # Rel.: #1311\n                # The target is invalid for a classification task if the feature type is string\n                # and not nominal. For such miss-configured tasks, we silently fix it here as\n                # we can safely interpreter string as nominal.\n                df, *_ = self.get_data()\n                return list(df[feature.name].unique())\n\n    return None\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.attributes_arff_from_df","title":"<code>attributes_arff_from_df(df)</code>","text":"<p>Describe attributes of the dataframe according to ARFF specification.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>(DataFrame, shape(n_samples, n_features))</code> <p>The dataframe containing the data set.</p> required <p>Returns:</p> Name Type Description <code>attributes_arff</code> <code>list[str]</code> <p>The data set attributes as required by the ARFF format.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def attributes_arff_from_df(df: pd.DataFrame) -&gt; list[tuple[str, list[str] | str]]:\n    \"\"\"Describe attributes of the dataframe according to ARFF specification.\n\n    Parameters\n    ----------\n    df : DataFrame, shape (n_samples, n_features)\n        The dataframe containing the data set.\n\n    Returns\n    -------\n    attributes_arff : list[str]\n        The data set attributes as required by the ARFF format.\n    \"\"\"\n    PD_DTYPES_TO_ARFF_DTYPE = {\"integer\": \"INTEGER\", \"floating\": \"REAL\", \"string\": \"STRING\"}\n    attributes_arff: list[tuple[str, list[str] | str]] = []\n\n    if not all(isinstance(column_name, str) for column_name in df.columns):\n        logger.warning(\"Converting non-str column names to str.\")\n        df.columns = [str(column_name) for column_name in df.columns]\n\n    for column_name in df:\n        # skipna=True does not infer properly the dtype. The NA values are\n        # dropped before the inference instead.\n        column_dtype = pd.api.types.infer_dtype(df[column_name].dropna(), skipna=False)\n\n        if column_dtype == \"categorical\":\n            # for categorical feature, arff expects a list string. However, a\n            # categorical column can contain mixed type and should therefore\n            # raise an error asking to convert all entries to string.\n            categories = df[column_name].cat.categories\n            categories_dtype = pd.api.types.infer_dtype(categories)\n            if categories_dtype not in (\"string\", \"unicode\"):\n                raise ValueError(\n                    f\"The column '{column_name}' of the dataframe is of \"\n                    \"'category' dtype. Therefore, all values in \"\n                    \"this columns should be string. Please \"\n                    \"convert the entries which are not string. \"\n                    f\"Got {categories_dtype} dtype in this column.\",\n                )\n            attributes_arff.append((column_name, categories.tolist()))\n        elif column_dtype == \"boolean\":\n            # boolean are encoded as categorical.\n            attributes_arff.append((column_name, [\"True\", \"False\"]))\n        elif column_dtype in PD_DTYPES_TO_ARFF_DTYPE:\n            attributes_arff.append((column_name, PD_DTYPES_TO_ARFF_DTYPE[column_dtype]))\n        else:\n            raise ValueError(\n                f\"The dtype '{column_dtype}' of the column '{column_name}' is not \"\n                \"currently supported by liac-arff. Supported \"\n                \"dtypes are categorical, string, integer, \"\n                \"floating, and boolean.\",\n            )\n    return attributes_arff\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.check_datasets_active","title":"<code>check_datasets_active(dataset_ids, raise_error_if_not_exist=True)</code>","text":"<p>Check if the dataset ids provided are active.</p> <p>Raises an error if a dataset_id in the given list of dataset_ids does not exist on the server and <code>raise_error_if_not_exist</code> is set to True (default).</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>List[int]</code> <p>A list of integers representing dataset ids.</p> required <code>raise_error_if_not_exist</code> <code>bool(default=True)</code> <p>Flag that if activated can raise an error, if one or more of the given dataset ids do not exist on the server.</p> <code>True</code> <p>Returns:</p> Type Description <code>dict</code> <p>A dictionary with items {did: bool}</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def check_datasets_active(\n    dataset_ids: list[int],\n    raise_error_if_not_exist: bool = True,  # noqa: FBT001, FBT002\n) -&gt; dict[int, bool]:\n    \"\"\"\n    Check if the dataset ids provided are active.\n\n    Raises an error if a dataset_id in the given list\n    of dataset_ids does not exist on the server and\n    `raise_error_if_not_exist` is set to True (default).\n\n    Parameters\n    ----------\n    dataset_ids : List[int]\n        A list of integers representing dataset ids.\n    raise_error_if_not_exist : bool (default=True)\n        Flag that if activated can raise an error, if one or more of the\n        given dataset ids do not exist on the server.\n\n    Returns\n    -------\n    dict\n        A dictionary with items {did: bool}\n    \"\"\"\n    datasets = list_datasets(status=\"all\", data_id=dataset_ids, output_format=\"dataframe\")\n    missing = set(dataset_ids) - set(datasets.get(\"did\", []))\n    if raise_error_if_not_exist and missing:\n        missing_str = \", \".join(str(did) for did in missing)\n        raise ValueError(f\"Could not find dataset(s) {missing_str} in OpenML dataset list.\")\n    return dict(datasets[\"status\"] == \"active\")\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.create_dataset","title":"<code>create_dataset(name, description, creator, contributor, collection_date, language, licence, attributes, data, default_target_attribute, ignore_attribute, citation, row_id_attribute=None, original_data_url=None, paper_url=None, update_comment=None, version_label=None)</code>","text":"<p>Create a dataset.</p> <p>This function creates an OpenMLDataset object. The OpenMLDataset object contains information related to the dataset and the actual data file.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> required <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> required <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> required <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> required <code>licence</code> <code>str</code> <p>License of the data.</p> required <code>attributes</code> <code>list, dict, or 'auto'</code> <p>A list of tuples. Each tuple consists of the attribute name and type. If passing a pandas DataFrame, the attributes can be automatically inferred by passing <code>'auto'</code>. Specific attributes can be manually specified by a passing a dictionary where the key is the name of the attribute and the value is the data type of the attribute.</p> required <code>data</code> <code>(ndarray, list, dataframe, coo_matrix, shape(n_samples, n_features))</code> <p>An array that contains both the attributes and the targets. When providing a dataframe, the attribute names and type can be inferred by passing <code>attributes='auto'</code>. The target feature is indicated as meta-data of the dataset.</p> required <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> required <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes. Can have multiple values, comma separated.</p> required <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> required <code>version_label</code> <code>str</code> <p>Version label provided by user.  Can be a date, hash, or some other type of id.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>class</code> <code>`openml.OpenMLDataset`</code> <code>Dataset description.</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def create_dataset(  # noqa: C901, PLR0912, PLR0915\n    name: str,\n    description: str | None,\n    creator: str | None,\n    contributor: str | None,\n    collection_date: str | None,\n    language: str | None,\n    licence: str | None,\n    # TODO(eddiebergman): Docstring says `type` but I don't know what this is other than strings\n    # Edit: Found it could also be like [\"True\", \"False\"]\n    attributes: list[tuple[str, str | list[str]]] | dict[str, str | list[str]] | Literal[\"auto\"],\n    data: pd.DataFrame | np.ndarray | scipy.sparse.coo_matrix,\n    # TODO(eddiebergman): Function requires `default_target_attribute` exist but API allows None\n    default_target_attribute: str,\n    ignore_attribute: str | list[str] | None,\n    citation: str,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n    update_comment: str | None = None,\n    version_label: str | None = None,\n) -&gt; OpenMLDataset:\n    \"\"\"Create a dataset.\n\n    This function creates an OpenMLDataset object.\n    The OpenMLDataset object contains information related to the dataset\n    and the actual data file.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str\n        License of the data.\n    attributes : list, dict, or 'auto'\n        A list of tuples. Each tuple consists of the attribute name and type.\n        If passing a pandas DataFrame, the attributes can be automatically\n        inferred by passing ``'auto'``. Specific attributes can be manually\n        specified by a passing a dictionary where the key is the name of the\n        attribute and the value is the data type of the attribute.\n    data : ndarray, list, dataframe, coo_matrix, shape (n_samples, n_features)\n        An array that contains both the attributes and the targets. When\n        providing a dataframe, the attribute names and type can be inferred by\n        passing ``attributes='auto'``.\n        The target feature is indicated as meta-data of the dataset.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n        Can have multiple values, comma separated.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    version_label : str, optional\n        Version label provided by user.\n         Can be a date, hash, or some other type of id.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n\n    Returns\n    -------\n    class:`openml.OpenMLDataset`\n    Dataset description.\n    \"\"\"\n    if isinstance(data, pd.DataFrame):\n        # infer the row id from the index of the dataset\n        if row_id_attribute is None:\n            row_id_attribute = data.index.name\n        # When calling data.values, the index will be skipped.\n        # We need to reset the index such that it is part of the data.\n        if data.index.name is not None:\n            data = data.reset_index()\n\n    if attributes == \"auto\" or isinstance(attributes, dict):\n        if not isinstance(data, pd.DataFrame):\n            raise ValueError(\n                \"Automatically inferring attributes requires \"\n                f\"a pandas DataFrame. A {data!r} was given instead.\",\n            )\n        # infer the type of data for each column of the DataFrame\n        attributes_ = attributes_arff_from_df(data)\n        if isinstance(attributes, dict):\n            # override the attributes which was specified by the user\n            for attr_idx in range(len(attributes_)):\n                attr_name = attributes_[attr_idx][0]\n                if attr_name in attributes:\n                    attributes_[attr_idx] = (attr_name, attributes[attr_name])\n    else:\n        attributes_ = attributes\n    ignore_attributes = _expand_parameter(ignore_attribute)\n    _validated_data_attributes(ignore_attributes, attributes_, \"ignore_attribute\")\n\n    default_target_attributes = _expand_parameter(default_target_attribute)\n    _validated_data_attributes(default_target_attributes, attributes_, \"default_target_attribute\")\n\n    if row_id_attribute is not None:\n        is_row_id_an_attribute = any(attr[0] == row_id_attribute for attr in attributes_)\n        if not is_row_id_an_attribute:\n            raise ValueError(\n                \"'row_id_attribute' should be one of the data attribute. \"\n                \" Got '{}' while candidates are {}.\".format(\n                    row_id_attribute,\n                    [attr[0] for attr in attributes_],\n                ),\n            )\n\n    if isinstance(data, pd.DataFrame):\n        if all(isinstance(dtype, pd.SparseDtype) for dtype in data.dtypes):\n            data = data.sparse.to_coo()\n            # liac-arff only support COO matrices with sorted rows\n            row_idx_sorted = np.argsort(data.row)  # type: ignore\n            data.row = data.row[row_idx_sorted]  # type: ignore\n            data.col = data.col[row_idx_sorted]  # type: ignore\n            data.data = data.data[row_idx_sorted]  # type: ignore\n        else:\n            data = data.to_numpy()\n\n    data_format: Literal[\"arff\", \"sparse_arff\"]\n    if isinstance(data, (list, np.ndarray)):\n        if isinstance(data[0], (list, np.ndarray)):\n            data_format = \"arff\"\n        elif isinstance(data[0], dict):\n            data_format = \"sparse_arff\"\n        else:\n            raise ValueError(\n                \"When giving a list or a numpy.ndarray, \"\n                \"they should contain a list/ numpy.ndarray \"\n                \"for dense data or a dictionary for sparse \"\n                f\"data. Got {data[0]!r} instead.\",\n            )\n    elif isinstance(data, coo_matrix):\n        data_format = \"sparse_arff\"\n    else:\n        raise ValueError(\n            \"When giving a list or a numpy.ndarray, \"\n            \"they should contain a list/ numpy.ndarray \"\n            \"for dense data or a dictionary for sparse \"\n            f\"data. Got {data[0]!r} instead.\",\n        )\n\n    arff_object = {\n        \"relation\": name,\n        \"description\": description,\n        \"attributes\": attributes_,\n        \"data\": data,\n    }\n\n    # serializes the ARFF dataset object and returns a string\n    arff_dataset = arff.dumps(arff_object)\n    try:\n        # check if ARFF is valid\n        decoder = arff.ArffDecoder()\n        return_type = arff.COO if data_format == \"sparse_arff\" else arff.DENSE\n        decoder.decode(arff_dataset, encode_nominal=True, return_type=return_type)\n    except arff.ArffException as e:\n        raise ValueError(\n            \"The arguments you have provided do not construct a valid ARFF file\"\n        ) from e\n\n    return OpenMLDataset(\n        name=name,\n        description=description,\n        data_format=data_format,\n        creator=creator,\n        contributor=contributor,\n        collection_date=collection_date,\n        language=language,\n        licence=licence,\n        default_target_attribute=default_target_attribute,\n        row_id_attribute=row_id_attribute,\n        ignore_attribute=ignore_attribute,\n        citation=citation,\n        version_label=version_label,\n        original_data_url=original_data_url,\n        paper_url=paper_url,\n        update_comment=update_comment,\n        dataset=arff_dataset,\n    )\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.delete_dataset","title":"<code>delete_dataset(dataset_id)</code>","text":"<p>Delete dataset with id <code>dataset_id</code> from the OpenML server.</p> <p>This can only be done if you are the owner of the dataset and no tasks are attached to the dataset.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int</code> <p>OpenML id of the dataset</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def delete_dataset(dataset_id: int) -&gt; bool:\n    \"\"\"Delete dataset with id `dataset_id` from the OpenML server.\n\n    This can only be done if you are the owner of the dataset and\n    no tasks are attached to the dataset.\n\n    Parameters\n    ----------\n    dataset_id : int\n        OpenML id of the dataset\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"data\", dataset_id)\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.edit_dataset","title":"<code>edit_dataset(data_id, description=None, creator=None, contributor=None, collection_date=None, language=None, default_target_attribute=None, ignore_attribute=None, citation=None, row_id_attribute=None, original_data_url=None, paper_url=None)</code>","text":"<p>Edits an OpenMLDataset.</p> <p>In addition to providing the dataset id of the dataset to edit (through data_id), you must specify a value for at least one of the optional function arguments, i.e. one value for a field to edit.</p> <p>This function allows editing of both non-critical and critical fields. Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.</p> <ul> <li>Editing non-critical data fields is allowed for all authenticated users.</li> <li>Editing critical fields is allowed only for the owner, provided there are no tasks    associated with this dataset.</li> </ul> <p>If dataset has tasks or if the user is not the owner, the only way to edit critical fields is to use fork_dataset followed by edit_dataset.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>ID of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dataset id</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def edit_dataset(\n    data_id: int,\n    description: str | None = None,\n    creator: str | None = None,\n    contributor: str | None = None,\n    collection_date: str | None = None,\n    language: str | None = None,\n    default_target_attribute: str | None = None,\n    ignore_attribute: str | list[str] | None = None,\n    citation: str | None = None,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n) -&gt; int:\n    \"\"\"Edits an OpenMLDataset.\n\n    In addition to providing the dataset id of the dataset to edit (through data_id),\n    you must specify a value for at least one of the optional function arguments,\n    i.e. one value for a field to edit.\n\n    This function allows editing of both non-critical and critical fields.\n    Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.\n\n     - Editing non-critical data fields is allowed for all authenticated users.\n     - Editing critical fields is allowed only for the owner, provided there are no tasks\n       associated with this dataset.\n\n    If dataset has tasks or if the user is not the owner, the only way\n    to edit critical fields is to use fork_dataset followed by edit_dataset.\n\n    Parameters\n    ----------\n    data_id : int\n        ID of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n\n    Returns\n    -------\n    Dataset id\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n\n    # compose data edit parameters as xml\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    xml = OrderedDict()  # type: 'OrderedDict[str, OrderedDict]'\n    xml[\"oml:data_edit_parameters\"] = OrderedDict()\n    xml[\"oml:data_edit_parameters\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n    xml[\"oml:data_edit_parameters\"][\"oml:description\"] = description\n    xml[\"oml:data_edit_parameters\"][\"oml:creator\"] = creator\n    xml[\"oml:data_edit_parameters\"][\"oml:contributor\"] = contributor\n    xml[\"oml:data_edit_parameters\"][\"oml:collection_date\"] = collection_date\n    xml[\"oml:data_edit_parameters\"][\"oml:language\"] = language\n    xml[\"oml:data_edit_parameters\"][\"oml:default_target_attribute\"] = default_target_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:row_id_attribute\"] = row_id_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:ignore_attribute\"] = ignore_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:citation\"] = citation\n    xml[\"oml:data_edit_parameters\"][\"oml:original_data_url\"] = original_data_url\n    xml[\"oml:data_edit_parameters\"][\"oml:paper_url\"] = paper_url\n\n    # delete None inputs\n    for k in list(xml[\"oml:data_edit_parameters\"]):\n        if not xml[\"oml:data_edit_parameters\"][k]:\n            del xml[\"oml:data_edit_parameters\"][k]\n\n    file_elements = {\n        \"edit_parameters\": (\"description.xml\", xmltodict.unparse(xml)),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        \"data/edit\",\n        \"post\",\n        data=form_data,\n        file_elements=file_elements,\n    )\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_edit\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.fork_dataset","title":"<code>fork_dataset(data_id)</code>","text":"<p>Creates a new dataset version, with the authenticated user as the new owner.  The forked dataset can have distinct dataset meta-data,  but the actual data itself is shared with the original version.</p> <p>This API is intended for use when a user is unable to edit the critical fields of a dataset  through the edit_dataset API.  (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)</p> <p>Specifically, this happens when the user is:         1. Not the owner of the dataset.         2. User is the owner of the dataset, but the dataset has tasks.</p> <p>In these two cases the only way to edit critical fields is:         1. STEP 1: Fork the dataset using fork_dataset API         2. STEP 2: Call edit_dataset API on the forked version.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to be forked</p> required <p>Returns:</p> Type Description <code>Dataset id of the forked dataset</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def fork_dataset(data_id: int) -&gt; int:\n    \"\"\"\n     Creates a new dataset version, with the authenticated user as the new owner.\n     The forked dataset can have distinct dataset meta-data,\n     but the actual data itself is shared with the original version.\n\n     This API is intended for use when a user is unable to edit the critical fields of a dataset\n     through the edit_dataset API.\n     (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)\n\n     Specifically, this happens when the user is:\n            1. Not the owner of the dataset.\n            2. User is the owner of the dataset, but the dataset has tasks.\n\n     In these two cases the only way to edit critical fields is:\n            1. STEP 1: Fork the dataset using fork_dataset API\n            2. STEP 2: Call edit_dataset API on the forked version.\n\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to be forked\n\n    Returns\n    -------\n    Dataset id of the forked dataset\n\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n    # compose data fork parameters\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"data/fork\", \"post\", data=form_data)\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_fork\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.get_dataset","title":"<code>get_dataset(dataset_id, download_data=None, version=None, error_if_multiple=False, cache_format='pickle', download_qualities=None, download_features_meta_data=None, download_all_files=False, force_refresh_cache=False)</code>","text":"<p>Download the OpenML dataset representation, optionally also download actual data file.</p> <p>This function is by default NOT thread/multiprocessing safe, as this function uses caching. A check will be performed to determine if the information has previously been downloaded to a cache, and if so be loaded from disk instead of retrieved from the server.</p> <p>To make this function thread safe, you can install the python package <code>oslo.concurrency</code>. If <code>oslo.concurrency</code> is installed <code>get_dataset</code> becomes thread safe.</p> <p>Alternatively, to make this function thread/multiprocessing safe initialize the cache first by calling <code>get_dataset(args)</code> once before calling <code>get_dataset(args)</code> many times in parallel. This will initialize the cache and later calls will use the cache in a thread/multiprocessing safe way.</p> <p>If dataset is retrieved by name, a version may be specified. If no version is specified and multiple versions of the dataset exist, the earliest version of the dataset that is still active will be returned. If no version is specified, multiple versions of the dataset exist and <code>exception_if_multiple</code> is set to <code>True</code>, this function will raise an exception.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int or str</code> <p>Dataset ID of the dataset to download</p> required <code>download_data</code> <code>bool(default=True)</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>None</code> <code>version</code> <code>(int, optional(default=None))</code> <p>Specifies the version if <code>dataset_id</code> is specified by name. If no version is specified, retrieve the least recent still active version.</p> <code>None</code> <code>error_if_multiple</code> <code>bool(default=False)</code> <p>If <code>True</code> raise an error if multiple datasets are found with matching criteria.</p> <code>False</code> <code>cache_format</code> <code>str(default='pickle') in {'pickle', 'feather'}</code> <p>Format for caching the dataset - may be feather or pickle Note that the default 'pickle' option may load slower than feather when no.of.rows is very high.</p> <code>'pickle'</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description. If True, download and cache the qualities file. If False, create the OpenMLDataset without qualities metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(qualities=True)</code> method.</p> <code>None</code> <code>download_features_meta_data</code> <code>bool(default=True)</code> <p>Option to download 'features' meta-data in addition to the minimal dataset description. If True, download and cache the features file. If False, create the OpenMLDataset without features metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(features=True)</code> method.</p> <code>None</code> <code>download_all_files</code> <code>bool</code> <p>EXPERIMENTAL. Download all files related to the dataset that reside on the server. Useful for datasets which refer to auxiliary files (e.g., meta-album).</p> <code>False</code> <code>force_refresh_cache</code> <code>bool(default=False)</code> <p>Force the cache to refreshed by deleting the cache directory and re-downloading the data. Note, if <code>force_refresh_cache</code> is True, <code>get_dataset</code> is NOT thread/multiprocessing safe, because this creates a race condition to creating and deleting the cache; as in general with the cache.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>dataset</code> <code>:class:`openml.OpenMLDataset`</code> <p>The downloaded dataset.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_dataset(  # noqa: C901, PLR0912\n    dataset_id: int | str,\n    download_data: bool | None = None,  # Optional for deprecation warning; later again only bool\n    version: int | None = None,\n    error_if_multiple: bool = False,  # noqa: FBT002, FBT001\n    cache_format: Literal[\"pickle\", \"feather\"] = \"pickle\",\n    download_qualities: bool | None = None,  # Same as above\n    download_features_meta_data: bool | None = None,  # Same as above\n    download_all_files: bool = False,  # noqa: FBT002, FBT001\n    force_refresh_cache: bool = False,  # noqa: FBT001, FBT002\n) -&gt; OpenMLDataset:\n    \"\"\"Download the OpenML dataset representation, optionally also download actual data file.\n\n    This function is by default NOT thread/multiprocessing safe, as this function uses caching.\n    A check will be performed to determine if the information has previously been downloaded to a\n    cache, and if so be loaded from disk instead of retrieved from the server.\n\n    To make this function thread safe, you can install the python package ``oslo.concurrency``.\n    If ``oslo.concurrency`` is installed `get_dataset` becomes thread safe.\n\n    Alternatively, to make this function thread/multiprocessing safe initialize the cache first by\n    calling `get_dataset(args)` once before calling `get_dataset(args)` many times in parallel.\n    This will initialize the cache and later calls will use the cache in a thread/multiprocessing\n    safe way.\n\n    If dataset is retrieved by name, a version may be specified.\n    If no version is specified and multiple versions of the dataset exist,\n    the earliest version of the dataset that is still active will be returned.\n    If no version is specified, multiple versions of the dataset exist and\n    ``exception_if_multiple`` is set to ``True``, this function will raise an exception.\n\n    Parameters\n    ----------\n    dataset_id : int or str\n        Dataset ID of the dataset to download\n    download_data : bool (default=True)\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    version : int, optional (default=None)\n        Specifies the version if `dataset_id` is specified by name.\n        If no version is specified, retrieve the least recent still active version.\n    error_if_multiple : bool (default=False)\n        If ``True`` raise an error if multiple datasets are found with matching criteria.\n    cache_format : str (default='pickle') in {'pickle', 'feather'}\n        Format for caching the dataset - may be feather or pickle\n        Note that the default 'pickle' option may load slower than feather when\n        no.of.rows is very high.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n        If True, download and cache the qualities file.\n        If False, create the OpenMLDataset without qualities metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(qualities=True)` method.\n    download_features_meta_data : bool (default=True)\n        Option to download 'features' meta-data in addition to the minimal dataset description.\n        If True, download and cache the features file.\n        If False, create the OpenMLDataset without features metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(features=True)` method.\n    download_all_files: bool (default=False)\n        EXPERIMENTAL. Download all files related to the dataset that reside on the server.\n        Useful for datasets which refer to auxiliary files (e.g., meta-album).\n    force_refresh_cache : bool (default=False)\n        Force the cache to refreshed by deleting the cache directory and re-downloading the data.\n        Note, if `force_refresh_cache` is True, `get_dataset` is NOT thread/multiprocessing safe,\n        because this creates a race condition to creating and deleting the cache; as in general with\n        the cache.\n\n    Returns\n    -------\n    dataset : :class:`openml.OpenMLDataset`\n        The downloaded dataset.\n    \"\"\"\n    # TODO(0.15): Remove the deprecation warning and make the default False; adjust types above\n    #   and documentation. Also remove None-to-True-cases below\n    if any(\n        download_flag is None\n        for download_flag in [download_data, download_qualities, download_features_meta_data]\n    ):\n        warnings.warn(\n            \"Starting from Version 0.15 `download_data`, `download_qualities`, and `download_featu\"\n            \"res_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy \"\n            \"loading. To disable this message until version 0.15 explicitly set `download_data`, \"\n            \"`download_qualities`, and `download_features_meta_data` to a bool while calling \"\n            \"`get_dataset`.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    download_data = True if download_data is None else download_data\n    download_qualities = True if download_qualities is None else download_qualities\n    download_features_meta_data = (\n        True if download_features_meta_data is None else download_features_meta_data\n    )\n\n    if download_all_files:\n        warnings.warn(\n            \"``download_all_files`` is experimental and is likely to break with new releases.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    if cache_format not in [\"feather\", \"pickle\"]:\n        raise ValueError(\n            \"cache_format must be one of 'feather' or 'pickle. \"\n            f\"Invalid format specified: {cache_format}\",\n        )\n\n    if isinstance(dataset_id, str):\n        try:\n            dataset_id = int(dataset_id)\n        except ValueError:\n            dataset_id = _name_to_id(dataset_id, version, error_if_multiple)  # type: ignore\n    elif not isinstance(dataset_id, int):\n        raise TypeError(\n            f\"`dataset_id` must be one of `str` or `int`, not {type(dataset_id)}.\",\n        )\n\n    if force_refresh_cache:\n        did_cache_dir = _get_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, dataset_id)\n        if did_cache_dir.exists():\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    did_cache_dir = _create_cache_directory_for_id(\n        DATASETS_CACHE_DIR_NAME,\n        dataset_id,\n    )\n\n    remove_dataset_cache = True\n    try:\n        description = _get_dataset_description(did_cache_dir, dataset_id)\n        features_file = None\n        qualities_file = None\n\n        if download_features_meta_data:\n            features_file = _get_dataset_features_file(did_cache_dir, dataset_id)\n        if download_qualities:\n            qualities_file = _get_dataset_qualities_file(did_cache_dir, dataset_id)\n\n        arff_file = _get_dataset_arff(description) if download_data else None\n        if \"oml:parquet_url\" in description and download_data:\n            try:\n                parquet_file = _get_dataset_parquet(\n                    description,\n                    download_all_files=download_all_files,\n                )\n            except urllib3.exceptions.MaxRetryError:\n                parquet_file = None\n            if parquet_file is None and arff_file:\n                logger.warning(\"Failed to download parquet, fallback on ARFF.\")\n        else:\n            parquet_file = None\n        remove_dataset_cache = False\n    except OpenMLServerException as e:\n        # if there was an exception\n        # check if the user had access to the dataset\n        if e.code == NO_ACCESS_GRANTED_ERRCODE:\n            raise OpenMLPrivateDatasetError(e.message) from None\n\n        raise e\n    finally:\n        if remove_dataset_cache:\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    return _create_dataset_from_description(\n        description,\n        features_file,\n        qualities_file,\n        arff_file,\n        parquet_file,\n        cache_format,\n    )\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.get_datasets","title":"<code>get_datasets(dataset_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download datasets.</p> <p>This function iterates :meth:<code>openml.datasets.get_dataset</code>.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>iterable</code> <p>Integers or strings representing dataset ids or dataset names. If dataset names are specified, the least recent still active dataset version is returned.</p> required <code>download_data</code> <code>bool</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>True</code> <code>download_qualities</code> <code>(bool, optional(default=True))</code> <p>If True, also download qualities.xml file. If False it skip the qualities.xml.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>list of datasets</code> <p>A list of dataset objects.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def get_datasets(\n    dataset_ids: list[str | int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLDataset]:\n    \"\"\"Download datasets.\n\n    This function iterates :meth:`openml.datasets.get_dataset`.\n\n    Parameters\n    ----------\n    dataset_ids : iterable\n        Integers or strings representing dataset ids or dataset names.\n        If dataset names are specified, the least recent still active dataset version is returned.\n    download_data : bool, optional\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    download_qualities : bool, optional (default=True)\n        If True, also download qualities.xml file. If False it skip the qualities.xml.\n\n    Returns\n    -------\n    datasets : list of datasets\n        A list of dataset objects.\n    \"\"\"\n    datasets = []\n    for dataset_id in dataset_ids:\n        datasets.append(\n            get_dataset(dataset_id, download_data, download_qualities=download_qualities),\n        )\n    return datasets\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.list_datasets","title":"<code>list_datasets(data_id=None, offset=None, size=None, status=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None, offset: int | None, size: int | None, status: str | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all dataset which are on OpenML. Supports large amount of results.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>list</code> <p>A list of data ids, to specify which datasets should be listed</p> <code>None</code> <code>offset</code> <code>int</code> <p>The number of datasets to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of datasets to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated}. By default active datasets are returned, but also datasets from another status can be requested.</p> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>output_format</code> <code>Literal['dataframe', 'dict']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators (keys in the dict): data_name, data_version, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from dataset ID to dict.</p> <p>Every dataset is represented by a dictionary containing the following information: - dataset id - name - format - status If qualities are calculated for the dataset, some of these are also returned.</p> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>dataset id</li> <li>name</li> <li>format</li> <li>status If qualities are calculated for the dataset, some of these are also included as columns.</li> </ul> </li> </ul> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_datasets(\n    data_id: list[int] | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dataframe\", \"dict\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all dataset which are on OpenML.\n    Supports large amount of results.\n\n    Parameters\n    ----------\n    data_id : list, optional\n        A list of data ids, to specify which datasets should be\n        listed\n    offset : int, optional\n        The number of datasets to skip, starting from the first.\n    size : int, optional\n        The maximum number of datasets to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated}. By\n        default active datasets are returned, but also datasets\n        from another status can be requested.\n    tag : str, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs : dict, optional\n        Legal filter operators (keys in the dict):\n        data_name, data_version, number_instances,\n        number_features, number_classes, number_missing_values.\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from dataset ID to dict.\n\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also included as columns.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        data_id=data_id,\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_datasets,\n        offset=offset,\n        size=size,\n        status=status,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.list_qualities","title":"<code>list_qualities()</code>","text":"<p>Return list of data qualities available.</p> <p>The function performs an API call to retrieve the entire list of data qualities that are computed on the datasets uploaded.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_qualities() -&gt; list[str]:\n    \"\"\"Return list of data qualities available.\n\n    The function performs an API call to retrieve the entire list of\n    data qualities that are computed on the datasets uploaded.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    api_call = \"data/qualities/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:quality\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:data_qualities_list\" not in qualities:\n        raise ValueError('Error in return XML, does not contain \"oml:data_qualities_list\"')\n\n    if not isinstance(qualities[\"oml:data_qualities_list\"][\"oml:quality\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:quality\" as a list')\n\n    return qualities[\"oml:data_qualities_list\"][\"oml:quality\"]\n</code></pre>"},{"location":"reference/datasets/#openml.datasets.status_update","title":"<code>status_update(data_id, status)</code>","text":"<p>Updates the status of a dataset to either 'active' or 'deactivated'. Please see the OpenML API documentation for a description of the status and all legal status transitions: https://docs.openml.org/#dataset-status</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/datasets/functions.py</code> <pre><code>def status_update(data_id: int, status: Literal[\"active\", \"deactivated\"]) -&gt; None:\n    \"\"\"\n    Updates the status of a dataset to either 'active' or 'deactivated'.\n    Please see the OpenML API documentation for a description of the status\n    and all legal status transitions:\n    https://docs.openml.org/#dataset-status\n\n    Parameters\n    ----------\n    data_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(f\"Illegal status value. Legal values: {legal_status}\")\n\n    data: openml._api_calls.DATA_TYPE = {\"data_id\": data_id, \"status\": status}\n    result_xml = openml._api_calls._perform_api_call(\"data/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_data_id = result[\"oml:data_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:data_status_update\"][\"oml:status\"]\n    if status != server_status or int(data_id) != int(server_data_id):\n        # This should never happen\n        raise ValueError(\"Data id/status does not collide\")\n</code></pre>"},{"location":"reference/datasets/data_feature/","title":"data_feature","text":""},{"location":"reference/datasets/data_feature/#openml.datasets.data_feature.OpenMLDataFeature","title":"<code>OpenMLDataFeature</code>","text":"<p>Data Feature (a.k.a. Attribute) object.</p> <p>Parameters:</p> Name Type Description Default <code>index</code> <code>int</code> <p>The index of this feature</p> required <code>name</code> <code>str</code> <p>Name of the feature</p> required <code>data_type</code> <code>str</code> <p>can be nominal, numeric, string, date (corresponds to arff)</p> required <code>nominal_values</code> <code>list(str)</code> <p>list of the possible values, in case of nominal attribute</p> required <code>number_missing_values</code> <code>int</code> <p>Number of rows that have a missing value for this feature.</p> required <code>ontologies</code> <code>list(str)</code> <p>list of ontologies attached to this feature. An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided.</p> <code>None</code> Source code in <code>openml/datasets/data_feature.py</code> <pre><code>class OpenMLDataFeature:\n    \"\"\"\n    Data Feature (a.k.a. Attribute) object.\n\n    Parameters\n    ----------\n    index : int\n        The index of this feature\n    name : str\n        Name of the feature\n    data_type : str\n        can be nominal, numeric, string, date (corresponds to arff)\n    nominal_values : list(str)\n        list of the possible values, in case of nominal attribute\n    number_missing_values : int\n        Number of rows that have a missing value for this feature.\n    ontologies : list(str)\n        list of ontologies attached to this feature. An ontology describes the\n        concept that are described in a feature. An ontology is defined by an\n        URL where the information is provided.\n    \"\"\"\n\n    LEGAL_DATA_TYPES: ClassVar[Sequence[str]] = [\"nominal\", \"numeric\", \"string\", \"date\"]\n\n    def __init__(  # noqa: PLR0913\n        self,\n        index: int,\n        name: str,\n        data_type: str,\n        nominal_values: list[str],\n        number_missing_values: int,\n        ontologies: list[str] | None = None,\n    ):\n        if not isinstance(index, int):\n            raise TypeError(f\"Index must be `int` but is {type(index)}\")\n\n        if data_type not in self.LEGAL_DATA_TYPES:\n            raise ValueError(\n                f\"data type should be in {self.LEGAL_DATA_TYPES!s}, found: {data_type}\",\n            )\n\n        if data_type == \"nominal\":\n            if nominal_values is None:\n                raise TypeError(\n                    \"Dataset features require attribute `nominal_values` for nominal \"\n                    \"feature type.\",\n                )\n\n            if not isinstance(nominal_values, list):\n                raise TypeError(\n                    \"Argument `nominal_values` is of wrong datatype, should be list, \"\n                    f\"but is {type(nominal_values)}\",\n                )\n        elif nominal_values is not None:\n            raise TypeError(\"Argument `nominal_values` must be None for non-nominal feature.\")\n\n        if not isinstance(number_missing_values, int):\n            msg = f\"number_missing_values must be int but is {type(number_missing_values)}\"\n            raise TypeError(msg)\n\n        self.index = index\n        self.name = str(name)\n        self.data_type = str(data_type)\n        self.nominal_values = nominal_values\n        self.number_missing_values = number_missing_values\n        self.ontologies = ontologies\n\n    def __repr__(self) -&gt; str:\n        return \"[%d - %s (%s)]\" % (self.index, self.name, self.data_type)\n\n    def __eq__(self, other: Any) -&gt; bool:\n        return isinstance(other, OpenMLDataFeature) and self.__dict__ == other.__dict__\n\n    def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -&gt; None:  # noqa: FBT001, ARG002\n        pp.text(str(self))\n</code></pre>"},{"location":"reference/datasets/dataset/","title":"dataset","text":""},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset","title":"<code>OpenMLDataset</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>Dataset object.</p> <p>Allows fetching and uploading datasets to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>data_format</code> <code>str</code> <p>Format of the dataset which can be either 'arff' or 'sparse_arff'.</p> <code>'arff'</code> <code>cache_format</code> <code>str</code> <p>Format for caching the dataset which can be either 'feather' or 'pickle'.</p> <code>'pickle'</code> <code>dataset_id</code> <code>int</code> <p>Id autogenerated by the server.</p> <code>None</code> <code>version</code> <code>int</code> <p>Version of this dataset. '1' for original version. Auto-incremented by server.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>The date-time when the dataset was uploaded, generated by server.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>licence</code> <code>str</code> <p>License of the data.</p> <code>None</code> <code>url</code> <code>str</code> <p>Valid URL, points to actual data file. The file can be on the OpenML server or another dataset repository.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>version_label</code> <code>str</code> <p>Version label provided by user. Can be a date, hash, or some other type of id.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>tag</code> <code>str</code> <p>Tags, describing the algorithms.</p> <code>None</code> <code>visibility</code> <code>str</code> <p>Who can see the dataset. Typical values: 'Everyone','All my friends','Only me'. Can also be any of the user's circles.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <code>md5_checksum</code> <code>str</code> <p>MD5 checksum to check if the dataset is downloaded without corruption.</p> <code>None</code> <code>data_file</code> <code>str</code> <p>Path to where the dataset is located.</p> <code>None</code> <code>features_file</code> <code>dict</code> <p>A dictionary of dataset features, which maps a feature index to a OpenMLDataFeature.</p> <code>None</code> <code>qualities_file</code> <code>dict</code> <p>A dictionary of dataset qualities, which maps a quality name to a quality value.</p> <code>None</code> <code>dataset</code> <code>str | None</code> <p>Serialized arff dataset string.</p> <code>None</code> <code>parquet_url</code> <code>str | None</code> <p>This is the URL to the storage location where the dataset files are hosted. This can be a MinIO bucket URL. If specified, the data will be accessed from this URL when reading the files.</p> <code>None</code> <code>parquet_file</code> <code>str | None</code> <p>Path to the local file.</p> <code>None</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>class OpenMLDataset(OpenMLBase):\n    \"\"\"Dataset object.\n\n    Allows fetching and uploading datasets to OpenML.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    data_format : str\n        Format of the dataset which can be either 'arff' or 'sparse_arff'.\n    cache_format : str\n        Format for caching the dataset which can be either 'feather' or 'pickle'.\n    dataset_id : int, optional\n        Id autogenerated by the server.\n    version : int, optional\n        Version of this dataset. '1' for original version.\n        Auto-incremented by server.\n    creator : str, optional\n        The person who created the dataset.\n    contributor : str, optional\n        People who contributed to the current version of the dataset.\n    collection_date : str, optional\n        The date the data was originally collected, given by the uploader.\n    upload_date : str, optional\n        The date-time when the dataset was uploaded, generated by server.\n    language : str, optional\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str, optional\n        License of the data.\n    url : str, optional\n        Valid URL, points to actual data file.\n        The file can be on the OpenML server or another dataset repository.\n    default_target_attribute : str, optional\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column,\n        if present in the dataset.\n    ignore_attribute : str | list, optional\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    version_label : str, optional\n        Version label provided by user.\n        Can be a date, hash, or some other type of id.\n    citation : str, optional\n        Reference(s) that should be cited when building on this data.\n    tag : str, optional\n        Tags, describing the algorithms.\n    visibility : str, optional\n        Who can see the dataset.\n        Typical values: 'Everyone','All my friends','Only me'.\n        Can also be any of the user's circles.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n    md5_checksum : str, optional\n        MD5 checksum to check if the dataset is downloaded without corruption.\n    data_file : str, optional\n        Path to where the dataset is located.\n    features_file : dict, optional\n        A dictionary of dataset features,\n        which maps a feature index to a OpenMLDataFeature.\n    qualities_file : dict, optional\n        A dictionary of dataset qualities,\n        which maps a quality name to a quality value.\n    dataset: string, optional\n        Serialized arff dataset string.\n    parquet_url: string, optional\n        This is the URL to the storage location where the dataset files are hosted.\n        This can be a MinIO bucket URL. If specified, the data will be accessed\n        from this URL when reading the files.\n    parquet_file: string, optional\n        Path to the local file.\n    \"\"\"\n\n    def __init__(  # noqa: C901, PLR0912, PLR0913, PLR0915\n        self,\n        name: str,\n        description: str | None,\n        data_format: Literal[\"arff\", \"sparse_arff\"] = \"arff\",\n        cache_format: Literal[\"feather\", \"pickle\"] = \"pickle\",\n        dataset_id: int | None = None,\n        version: int | None = None,\n        creator: str | None = None,\n        contributor: str | None = None,\n        collection_date: str | None = None,\n        upload_date: str | None = None,\n        language: str | None = None,\n        licence: str | None = None,\n        url: str | None = None,\n        default_target_attribute: str | None = None,\n        row_id_attribute: str | None = None,\n        ignore_attribute: str | list[str] | None = None,\n        version_label: str | None = None,\n        citation: str | None = None,\n        tag: str | None = None,\n        visibility: str | None = None,\n        original_data_url: str | None = None,\n        paper_url: str | None = None,\n        update_comment: str | None = None,\n        md5_checksum: str | None = None,\n        data_file: str | None = None,\n        features_file: str | None = None,\n        qualities_file: str | None = None,\n        dataset: str | None = None,\n        parquet_url: str | None = None,\n        parquet_file: str | None = None,\n    ):\n        if cache_format not in [\"feather\", \"pickle\"]:\n            raise ValueError(\n                \"cache_format must be one of 'feather' or 'pickle. \"\n                f\"Invalid format specified: {cache_format}\",\n            )\n\n        def find_invalid_characters(string: str, pattern: str) -&gt; str:\n            invalid_chars = set()\n            regex = re.compile(pattern)\n            for char in string:\n                if not regex.match(char):\n                    invalid_chars.add(char)\n            return \",\".join(\n                [f\"'{char}'\" if char != \"'\" else f'\"{char}\"' for char in invalid_chars],\n            )\n\n        if dataset_id is None:\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if description and not re.match(pattern, description):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(description, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in description: {description}\",\n                )\n            pattern = \"^[\\x00-\\x7F]*$\"\n            if citation and not re.match(pattern, citation):\n                # not basiclatin (XSD complains)\n                invalid_characters = find_invalid_characters(citation, pattern)\n                raise ValueError(\n                    f\"Invalid symbols {invalid_characters} in citation: {citation}\",\n                )\n            pattern = \"^[a-zA-Z0-9_\\\\-\\\\.\\\\(\\\\),]+$\"\n            if not re.match(pattern, name):\n                # regex given by server in error message\n                invalid_characters = find_invalid_characters(name, pattern)\n                raise ValueError(f\"Invalid symbols {invalid_characters} in name: {name}\")\n\n        self.ignore_attribute: list[str] | None = None\n        if isinstance(ignore_attribute, str):\n            self.ignore_attribute = [ignore_attribute]\n        elif isinstance(ignore_attribute, list) or ignore_attribute is None:\n            self.ignore_attribute = ignore_attribute\n        else:\n            raise ValueError(\"Wrong data type for ignore_attribute. Should be list.\")\n\n        # TODO add function to check if the name is casual_string128\n        # Attributes received by querying the RESTful API\n        self.dataset_id = int(dataset_id) if dataset_id is not None else None\n        self.name = name\n        self.version = int(version) if version is not None else None\n        self.description = description\n        self.cache_format = cache_format\n        # Has to be called format, otherwise there will be an XML upload error\n        self.format = data_format\n        self.creator = creator\n        self.contributor = contributor\n        self.collection_date = collection_date\n        self.upload_date = upload_date\n        self.language = language\n        self.licence = licence\n        self.url = url\n        self.default_target_attribute = default_target_attribute\n        self.row_id_attribute = row_id_attribute\n\n        self.version_label = version_label\n        self.citation = citation\n        self.tag = tag\n        self.visibility = visibility\n        self.original_data_url = original_data_url\n        self.paper_url = paper_url\n        self.update_comment = update_comment\n        self.md5_checksum = md5_checksum\n        self.data_file = data_file\n        self.parquet_file = parquet_file\n        self._dataset = dataset\n        self._parquet_url = parquet_url\n\n        self._features: dict[int, OpenMLDataFeature] | None = None\n        self._qualities: dict[str, float] | None = None\n        self._no_qualities_found = False\n\n        if features_file is not None:\n            self._features = _read_features(Path(features_file))\n\n        # \"\" was the old default value by `get_dataset` and maybe still used by some\n        if qualities_file == \"\":\n            # TODO(0.15): to switch to \"qualities_file is not None\" below and remove warning\n            warnings.warn(\n                \"Starting from Version 0.15 `qualities_file` must be None and not an empty string \"\n                \"to avoid reading the qualities from file. Set `qualities_file` to None to avoid \"\n                \"this warning.\",\n                FutureWarning,\n                stacklevel=2,\n            )\n            qualities_file = None\n\n        if qualities_file is not None:\n            self._qualities = _read_qualities(Path(qualities_file))\n\n        if data_file is not None:\n            data_pickle, data_feather, feather_attribute = self._compressed_cache_file_paths(\n                Path(data_file)\n            )\n            self.data_pickle_file = data_pickle if Path(data_pickle).exists() else None\n            self.data_feather_file = data_feather if Path(data_feather).exists() else None\n            self.feather_attribute_file = feather_attribute if Path(feather_attribute) else None\n        else:\n            self.data_pickle_file = None\n            self.data_feather_file = None\n            self.feather_attribute_file = None\n\n    @property\n    def features(self) -&gt; dict[int, OpenMLDataFeature]:\n        \"\"\"Get the features of this dataset.\"\"\"\n        if self._features is None:\n            # TODO(eddiebergman): These should return a value so we can set it to be not None\n            self._load_features()\n\n        assert self._features is not None\n        return self._features\n\n    @property\n    def qualities(self) -&gt; dict[str, float] | None:\n        \"\"\"Get the qualities of this dataset.\"\"\"\n        # TODO(eddiebergman): Better docstring, I don't know what qualities means\n\n        # We have to check `_no_qualities_found` as there might not be qualities for a dataset\n        if self._qualities is None and (not self._no_qualities_found):\n            self._load_qualities()\n\n        return self._qualities\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Get the dataset numeric id.\"\"\"\n        return self.dataset_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | None]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Obtain number of features in accordance with lazy loading.\n        n_features: int | None = None\n        if self._qualities is not None and self._qualities[\"NumberOfFeatures\"] is not None:\n            n_features = int(self._qualities[\"NumberOfFeatures\"])\n        elif self._features is not None:\n            n_features = len(self._features)\n\n        fields: dict[str, int | str | None] = {\n            \"Name\": self.name,\n            \"Version\": self.version,\n            \"Format\": self.format,\n            \"Licence\": self.licence,\n            \"Download URL\": self.url,\n            \"Data file\": str(self.data_file) if self.data_file is not None else None,\n            \"Pickle file\": (\n                str(self.data_pickle_file) if self.data_pickle_file is not None else None\n            ),\n            \"# of features\": n_features,\n        }\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.dataset_id is not None:\n            fields[\"OpenML URL\"] = self.openml_url\n        if self._qualities is not None and self._qualities[\"NumberOfInstances\"] is not None:\n            fields[\"# of instances\"] = int(self._qualities[\"NumberOfInstances\"])\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Name\",\n            \"Version\",\n            \"Format\",\n            \"Upload Date\",\n            \"Licence\",\n            \"Download URL\",\n            \"OpenML URL\",\n            \"Data File\",\n            \"Pickle File\",\n            \"# of features\",\n            \"# of instances\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if not isinstance(other, OpenMLDataset):\n            return False\n\n        server_fields = {\n            \"dataset_id\",\n            \"version\",\n            \"upload_date\",\n            \"url\",\n            \"dataset\",\n            \"data_file\",\n        }\n\n        # check that common keys and values are identical\n        self_keys = set(self.__dict__.keys()) - server_fields\n        other_keys = set(other.__dict__.keys()) - server_fields\n        return self_keys == other_keys and all(\n            self.__dict__[key] == other.__dict__[key] for key in self_keys\n        )\n\n    def _download_data(self) -&gt; None:\n        \"\"\"Download ARFF data file to standard cache directory. Set `self.data_file`.\"\"\"\n        # import required here to avoid circular import.\n        from .functions import _get_dataset_arff, _get_dataset_parquet\n\n        self.data_file = str(_get_dataset_arff(self))\n        if self._parquet_url is not None:\n            self.parquet_file = str(_get_dataset_parquet(self))\n\n    def _get_arff(self, format: str) -&gt; dict:  # noqa: A002\n        \"\"\"Read ARFF file and return decoded arff.\n\n        Reads the file referenced in self.data_file.\n\n        Parameters\n        ----------\n        format : str\n            Format of the ARFF file.\n            Must be one of 'arff' or 'sparse_arff' or a string that will be either of those\n            when converted to lower case.\n\n\n\n        Returns\n        -------\n        dict\n            Decoded arff.\n\n        \"\"\"\n        # TODO: add a partial read method which only returns the attribute\n        # headers of the corresponding .arff file!\n        import struct\n\n        filename = self.data_file\n        assert filename is not None\n        filepath = Path(filename)\n\n        bits = 8 * struct.calcsize(\"P\")\n\n        # Files can be considered too large on a 32-bit system,\n        # if it exceeds 120mb (slightly more than covtype dataset size)\n        # This number is somewhat arbitrary.\n        if bits != 64:\n            MB_120 = 120_000_000\n            file_size = filepath.stat().st_size\n            if file_size &gt; MB_120:\n                raise NotImplementedError(\n                    f\"File {filename} too big for {file_size}-bit system ({bits} bytes).\",\n                )\n\n        if format.lower() == \"arff\":\n            return_type = arff.DENSE\n        elif format.lower() == \"sparse_arff\":\n            return_type = arff.COO\n        else:\n            raise ValueError(f\"Unknown data format {format}\")\n\n        def decode_arff(fh: Any) -&gt; dict:\n            decoder = arff.ArffDecoder()\n            return decoder.decode(fh, encode_nominal=True, return_type=return_type)  # type: ignore\n\n        if filepath.suffix.endswith(\".gz\"):\n            with gzip.open(filename) as zipfile:\n                return decode_arff(zipfile)\n        else:\n            with filepath.open(encoding=\"utf8\") as fh:\n                return decode_arff(fh)\n\n    def _parse_data_from_arff(  # noqa: C901, PLR0912, PLR0915\n        self,\n        arff_file_path: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Parse all required data from arff file.\n\n        Parameters\n        ----------\n        arff_file_path : str\n            Path to the file on disk.\n\n        Returns\n        -------\n        Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]\n            DataFrame or csr_matrix: dataset\n            List[bool]: List indicating which columns contain categorical variables.\n            List[str]: List of column names.\n        \"\"\"\n        try:\n            data = self._get_arff(self.format)\n        except OSError as e:\n            logger.critical(\n                f\"Please check that the data file {arff_file_path} is \" \"there and can be read.\",\n            )\n            raise e\n\n        ARFF_DTYPES_TO_PD_DTYPE = {\n            \"INTEGER\": \"integer\",\n            \"REAL\": \"floating\",\n            \"NUMERIC\": \"floating\",\n            \"STRING\": \"string\",\n        }\n        attribute_dtype = {}\n        attribute_names = []\n        categories_names = {}\n        categorical = []\n        for name, type_ in data[\"attributes\"]:\n            # if the feature is nominal and a sparse matrix is\n            # requested, the categories need to be numeric\n            if isinstance(type_, list) and self.format.lower() == \"sparse_arff\":\n                try:\n                    # checks if the strings which should be the class labels\n                    # can be encoded into integers\n                    pd.factorize(type_)[0]\n                except ValueError as e:\n                    raise ValueError(\n                        \"Categorical data needs to be numeric when using sparse ARFF.\"\n                    ) from e\n\n            # string can only be supported with pandas DataFrame\n            elif type_ == \"STRING\" and self.format.lower() == \"sparse_arff\":\n                raise ValueError(\"Dataset containing strings is not supported with sparse ARFF.\")\n\n            # infer the dtype from the ARFF header\n            if isinstance(type_, list):\n                categorical.append(True)\n                categories_names[name] = type_\n                if len(type_) == 2:\n                    type_norm = [cat.lower().capitalize() for cat in type_]\n                    if {\"True\", \"False\"} == set(type_norm):\n                        categories_names[name] = [cat == \"True\" for cat in type_norm]\n                        attribute_dtype[name] = \"boolean\"\n                    else:\n                        attribute_dtype[name] = \"categorical\"\n                else:\n                    attribute_dtype[name] = \"categorical\"\n            else:\n                categorical.append(False)\n                attribute_dtype[name] = ARFF_DTYPES_TO_PD_DTYPE[type_]\n            attribute_names.append(name)\n\n        if self.format.lower() == \"sparse_arff\":\n            X = data[\"data\"]\n            X_shape = (max(X[1]) + 1, max(X[2]) + 1)\n            X = scipy.sparse.coo_matrix((X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32)\n            X = X.tocsr()\n        elif self.format.lower() == \"arff\":\n            X = pd.DataFrame(data[\"data\"], columns=attribute_names)\n\n            col = []\n            for column_name in X.columns:\n                if attribute_dtype[column_name] in (\"categorical\", \"boolean\"):\n                    categories = self._unpack_categories(\n                        X[column_name],  # type: ignore\n                        categories_names[column_name],\n                    )\n                    col.append(categories)\n                elif attribute_dtype[column_name] in (\"floating\", \"integer\"):\n                    X_col = X[column_name]\n                    if X_col.min() &gt;= 0 and X_col.max() &lt;= 255:\n                        try:\n                            X_col_uint = X_col.astype(\"uint8\")\n                            if (X_col == X_col_uint).all():\n                                col.append(X_col_uint)\n                                continue\n                        except ValueError:\n                            pass\n                    col.append(X[column_name])\n                else:\n                    col.append(X[column_name])\n            X = pd.concat(col, axis=1)\n        else:\n            raise ValueError(f\"Dataset format '{self.format}' is not a valid format.\")\n\n        return X, categorical, attribute_names  # type: ignore\n\n    def _compressed_cache_file_paths(self, data_file: Path) -&gt; tuple[Path, Path, Path]:\n        data_pickle_file = data_file.with_suffix(\".pkl.py3\")\n        data_feather_file = data_file.with_suffix(\".feather\")\n        feather_attribute_file = data_file.with_suffix(\".feather.attributes.pkl.py3\")\n        return data_pickle_file, data_feather_file, feather_attribute_file\n\n    def _cache_compressed_file_from_file(\n        self,\n        data_file: Path,\n    ) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:\n        \"\"\"Store data from the local file in compressed format.\n\n        If a local parquet file is present it will be used instead of the arff file.\n        Sets cache_format to 'pickle' if data is sparse.\n        \"\"\"\n        (\n            data_pickle_file,\n            data_feather_file,\n            feather_attribute_file,\n        ) = self._compressed_cache_file_paths(data_file)\n\n        if data_file.suffix == \".arff\":\n            data, categorical, attribute_names = self._parse_data_from_arff(data_file)\n        elif data_file.suffix == \".pq\":\n            try:\n                data = pd.read_parquet(data_file)\n            except Exception as e:  # noqa: BLE001\n                raise Exception(f\"File: {data_file}\") from e\n\n            categorical = [data[c].dtype.name == \"category\" for c in data.columns]\n            attribute_names = list(data.columns)\n        else:\n            raise ValueError(f\"Unknown file type for file '{data_file}'.\")\n\n        # Feather format does not work for sparse datasets, so we use pickle for sparse datasets\n        if scipy.sparse.issparse(data):\n            self.cache_format = \"pickle\"\n\n        logger.info(f\"{self.cache_format} write {self.name}\")\n        if self.cache_format == \"feather\":\n            assert isinstance(data, pd.DataFrame)\n\n            data.to_feather(data_feather_file)\n            with open(feather_attribute_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_feather_file = data_feather_file\n            self.feather_attribute_file = feather_attribute_file\n\n        else:\n            with open(data_pickle_file, \"wb\") as fh:  # noqa: PTH123\n                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)\n            self.data_pickle_file = data_pickle_file\n\n        data_file = data_pickle_file if self.cache_format == \"pickle\" else data_feather_file\n        logger.debug(f\"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}\")\n\n        return data, categorical, attribute_names\n\n    def _load_data(self) -&gt; tuple[pd.DataFrame | scipy.sparse.csr_matrix, list[bool], list[str]]:  # noqa: PLR0912, C901\n        \"\"\"Load data from compressed format or arff. Download data if not present on disk.\"\"\"\n        need_to_create_pickle = self.cache_format == \"pickle\" and self.data_pickle_file is None\n        need_to_create_feather = self.cache_format == \"feather\" and self.data_feather_file is None\n\n        if need_to_create_pickle or need_to_create_feather:\n            if self.data_file is None:\n                self._download_data()\n\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n\n        # helper variable to help identify where errors occur\n        fpath = self.data_feather_file if self.cache_format == \"feather\" else self.data_pickle_file\n        logger.info(f\"{self.cache_format} load data {self.name}\")\n        try:\n            if self.cache_format == \"feather\":\n                assert self.data_feather_file is not None\n                assert self.feather_attribute_file is not None\n\n                data = pd.read_feather(self.data_feather_file)\n                fpath = self.feather_attribute_file\n                with open(self.feather_attribute_file, \"rb\") as fh:  # noqa: PTH123\n                    categorical, attribute_names = pickle.load(fh)  # noqa: S301\n            else:\n                assert self.data_pickle_file is not None\n                with open(self.data_pickle_file, \"rb\") as fh:  # noqa: PTH123\n                    data, categorical, attribute_names = pickle.load(fh)  # noqa: S301\n        except FileNotFoundError as e:\n            raise ValueError(\n                f\"Cannot find file for dataset {self.name} at location '{fpath}'.\"\n            ) from e\n        except (EOFError, ModuleNotFoundError, ValueError, AttributeError) as e:\n            error_message = getattr(e, \"message\", e.args[0])\n            hint = \"\"\n\n            if isinstance(e, EOFError):\n                readable_error = \"Detected a corrupt cache file\"\n            elif isinstance(e, (ModuleNotFoundError, AttributeError)):\n                readable_error = \"Detected likely dependency issues\"\n                hint = (\n                    \"This can happen if the cache was constructed with a different pandas version \"\n                    \"than the one that is used to load the data. See also \"\n                )\n                if isinstance(e, ModuleNotFoundError):\n                    hint += \"https://github.com/openml/openml-python/issues/918. \"\n                elif isinstance(e, AttributeError):\n                    hint += \"https://github.com/openml/openml-python/pull/1121. \"\n\n            elif isinstance(e, ValueError) and \"unsupported pickle protocol\" in e.args[0]:\n                readable_error = \"Encountered unsupported pickle protocol\"\n            else:\n                raise e\n\n            logger.warning(\n                f\"{readable_error} when loading dataset {self.id} from '{fpath}'. \"\n                f\"{hint}\"\n                f\"Error message was: {error_message}. \"\n                \"We will continue loading data from the arff-file, \"\n                \"but this will be much slower for big datasets. \"\n                \"Please manually delete the cache file if you want OpenML-Python \"\n                \"to attempt to reconstruct it.\",\n            )\n            assert self.data_file is not None\n            data, categorical, attribute_names = self._parse_data_from_arff(Path(self.data_file))\n\n        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)\n        if self.cache_format == \"pickle\" and not data_up_to_date:\n            logger.info(\"Updating outdated pickle file.\")\n            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file\n            assert file_to_load is not None\n\n            return self._cache_compressed_file_from_file(Path(file_to_load))\n        return data, categorical, attribute_names\n\n    # TODO(eddiebergman): Can type this better with overload\n    # TODO(eddiebergman): Could also techinically use scipy.sparse.sparray\n    @staticmethod\n    def _convert_array_format(\n        data: pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix,\n        array_format: Literal[\"array\", \"dataframe\"],\n        attribute_names: list | None = None,\n    ) -&gt; pd.DataFrame | pd.Series | np.ndarray | scipy.sparse.spmatrix:\n        \"\"\"Convert a dataset to a given array format.\n\n        Converts to numpy array if data is non-sparse.\n        Converts to a sparse dataframe if data is sparse.\n\n        Parameters\n        ----------\n        array_format : str {'array', 'dataframe'}\n            Desired data type of the output\n            - If array_format='array'\n                If data is non-sparse\n                    Converts to numpy-array\n                    Enforces numeric encoding of categorical columns\n                    Missing values are represented as NaN in the numpy-array\n                else returns data as is\n            - If array_format='dataframe'\n                If data is sparse\n                    Works only on sparse data\n                    Converts sparse data to sparse dataframe\n                else returns data as is\n\n        \"\"\"\n        if array_format == \"array\" and not isinstance(data, scipy.sparse.spmatrix):\n            # We encode the categories such that they are integer to be able\n            # to make a conversion to numeric for backward compatibility\n            def _encode_if_category(column: pd.Series | np.ndarray) -&gt; pd.Series | np.ndarray:\n                if column.dtype.name == \"category\":\n                    column = column.cat.codes.astype(np.float32)\n                    mask_nan = column == -1\n                    column[mask_nan] = np.nan\n                return column\n\n            if isinstance(data, pd.DataFrame):\n                columns = {\n                    column_name: _encode_if_category(data.loc[:, column_name])\n                    for column_name in data.columns\n                }\n                data = pd.DataFrame(columns)\n            else:\n                data = _encode_if_category(data)\n\n            try:\n                # TODO(eddiebergman): float32?\n                return_array = np.asarray(data, dtype=np.float32)\n            except ValueError as e:\n                raise PyOpenMLError(\n                    \"PyOpenML cannot handle string when returning numpy\"\n                    ' arrays. Use dataset_format=\"dataframe\".',\n                ) from e\n\n            return return_array\n\n        if array_format == \"dataframe\":\n            if scipy.sparse.issparse(data):\n                data = pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)\n        else:\n            data_type = \"sparse-data\" if scipy.sparse.issparse(data) else \"non-sparse data\"\n            logger.warning(\n                f\"Cannot convert {data_type} ({type(data)}) to '{array_format}'.\"\n                \" Returning input data.\",\n            )\n        return data\n\n    @staticmethod\n    def _unpack_categories(series: pd.Series, categories: list) -&gt; pd.Series:\n        # nan-likes can not be explicitly specified as a category\n        def valid_category(cat: Any) -&gt; bool:\n            return isinstance(cat, str) or (cat is not None and not np.isnan(cat))\n\n        filtered_categories = [c for c in categories if valid_category(c)]\n        col = []\n        for x in series:\n            try:\n                col.append(categories[int(x)])\n            except (TypeError, ValueError):\n                col.append(np.nan)\n\n        # We require two lines to create a series of categories as detailed here:\n        # https://pandas.pydata.org/pandas-docs/version/0.24/user_guide/categorical.html#series-creation\n        raw_cat = pd.Categorical(col, ordered=True, categories=filtered_categories)\n        return pd.Series(raw_cat, index=series.index, name=series.name)\n\n    def get_data(  # noqa: C901, PLR0912, PLR0915\n        self,\n        target: list[str] | str | None = None,\n        include_row_id: bool = False,  # noqa: FBT001, FBT002\n        include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n        np.ndarray | pd.DataFrame | None,\n        list[bool],\n        list[str],\n    ]:\n        \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n        Parameters\n        ----------\n        target : string, List[str] or None (default=None)\n            Name of target column to separate from the data.\n            Splitting multiple columns is currently not supported.\n        include_row_id : boolean (default=False)\n            Whether to include row ids in the returned dataset.\n        include_ignore_attribute : boolean (default=False)\n            Whether to include columns that are marked as \"ignore\"\n            on the server in the dataset.\n        dataset_format : string (default='dataframe')\n            The format of returned dataset.\n            If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n            matrix. Support for ``array`` will be removed in 0.15.\n            If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n        Returns\n        -------\n        X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n            Dataset\n        y : ndarray or pd.Series, shape (n_samples, ) or None\n            Target column\n        categorical_indicator : boolean ndarray\n            Mask that indicate categorical features.\n        attribute_names : List[str]\n            List of attribute names.\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        data, categorical, attribute_names = self._load_data()\n\n        to_exclude = []\n        if not include_row_id and self.row_id_attribute is not None:\n            if isinstance(self.row_id_attribute, str):\n                to_exclude.append(self.row_id_attribute)\n            elif isinstance(self.row_id_attribute, Iterable):\n                to_exclude.extend(self.row_id_attribute)\n\n        if not include_ignore_attribute and self.ignore_attribute is not None:\n            if isinstance(self.ignore_attribute, str):\n                to_exclude.append(self.ignore_attribute)\n            elif isinstance(self.ignore_attribute, Iterable):\n                to_exclude.extend(self.ignore_attribute)\n\n        if len(to_exclude) &gt; 0:\n            logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n            keep = np.array([column not in to_exclude for column in attribute_names])\n            data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n            categorical = [cat for cat, k in zip(categorical, keep) if k]\n            attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n        if target is None:\n            data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n            targets = None\n        else:\n            if isinstance(target, str):\n                target = target.split(\",\") if \",\" in target else [target]\n            targets = np.array([column in target for column in attribute_names])\n            target_names = [column for column in attribute_names if column in target]\n            if np.sum(targets) &gt; 1:\n                raise NotImplementedError(\n                    \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n                )\n            target_categorical = [\n                cat for cat, column in zip(categorical, attribute_names) if column in target\n            ]\n            target_dtype = int if target_categorical[0] else float\n\n            if isinstance(data, pd.DataFrame):\n                x = data.iloc[:, ~targets]\n                y = data.iloc[:, targets]\n            else:\n                x = data[:, ~targets]\n                y = data[:, targets].astype(target_dtype)  # type: ignore\n\n            categorical = [cat for cat, t in zip(categorical, targets) if not t]\n            attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n            x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n            if dataset_format == \"array\" and scipy.sparse.issparse(y):\n                # scikit-learn requires dense representation of targets\n                y = np.asarray(y.todense()).astype(target_dtype)\n                # dense representation of single column sparse arrays become a 2-d array\n                # need to flatten it to a 1-d array for _convert_array_format()\n                y = y.squeeze()\n            y = self._convert_array_format(y, dataset_format, target_names)\n            y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n            if len(y.shape) &gt; 1 and y.shape[1] == 1:\n                # single column targets should be 1-d for both `array` and `dataframe` formats\n                assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n                y = y.squeeze()\n            data, targets = x, y\n\n        return data, targets, categorical, attribute_names  # type: ignore\n\n    def _load_features(self) -&gt; None:\n        \"\"\"Load the features metadata from the server and store it in the dataset object.\"\"\"\n        # Delayed Import to avoid circular imports or having to import all of dataset.functions to\n        # import OpenMLDataset.\n        from openml.datasets.functions import _get_dataset_features_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        features_file = _get_dataset_features_file(None, self.dataset_id)\n        self._features = _read_features(features_file)\n\n    def _load_qualities(self) -&gt; None:\n        \"\"\"Load qualities information from the server and store it in the dataset object.\"\"\"\n        # same reason as above for _load_features\n        from openml.datasets.functions import _get_dataset_qualities_file\n\n        if self.dataset_id is None:\n            raise ValueError(\n                \"No dataset id specified. Please set the dataset id. Otherwise we cannot load \"\n                \"metadata.\",\n            )\n\n        qualities_file = _get_dataset_qualities_file(None, self.dataset_id)\n\n        if qualities_file is None:\n            self._no_qualities_found = True\n        else:\n            self._qualities = _read_qualities(qualities_file)\n\n    def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n        \"\"\"Reads the datasets arff to determine the class-labels.\n\n        If the task has no class labels (for example a regression problem)\n        it returns None. Necessary because the data returned by get_data\n        only contains the indices of the classes, while OpenML needs the real\n        classname when uploading the results of a run.\n\n        Parameters\n        ----------\n        target_name : str\n            Name of the target attribute\n\n        Returns\n        -------\n        list\n        \"\"\"\n        for feature in self.features.values():\n            if feature.name == target_name:\n                if feature.data_type == \"nominal\":\n                    return feature.nominal_values\n\n                if feature.data_type == \"string\":\n                    # Rel.: #1311\n                    # The target is invalid for a classification task if the feature type is string\n                    # and not nominal. For such miss-configured tasks, we silently fix it here as\n                    # we can safely interpreter string as nominal.\n                    df, *_ = self.get_data()\n                    return list(df[feature.name].unique())\n\n        return None\n\n    def get_features_by_type(  # noqa: C901\n        self,\n        data_type: str,\n        exclude: list[str] | None = None,\n        exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n        exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; list[int]:\n        \"\"\"\n        Return indices of features of a given type, e.g. all nominal features.\n        Optional parameters to exclude various features by index or ontology.\n\n        Parameters\n        ----------\n        data_type : str\n            The data type to return (e.g., nominal, numeric, date, string)\n        exclude : list(int)\n            List of columns to exclude from the return value\n        exclude_ignore_attribute : bool\n            Whether to exclude the defined ignore attributes (and adapt the\n            return values as if these indices are not present)\n        exclude_row_id_attribute : bool\n            Whether to exclude the defined row id attributes (and adapt the\n            return values as if these indices are not present)\n\n        Returns\n        -------\n        result : list\n            a list of indices that have the specified data type\n        \"\"\"\n        if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n            raise TypeError(\"Illegal feature type requested\")\n        if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n            raise TypeError(\"ignore_attribute should be a list\")\n        if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n            raise TypeError(\"row id attribute should be a str\")\n        if exclude is not None and not isinstance(exclude, list):\n            raise TypeError(\"Exclude should be a list\")\n            # assert all(isinstance(elem, str) for elem in exclude),\n            #            \"Exclude should be a list of strings\"\n        to_exclude = []\n        if exclude is not None:\n            to_exclude.extend(exclude)\n        if exclude_ignore_attribute and self.ignore_attribute is not None:\n            to_exclude.extend(self.ignore_attribute)\n        if exclude_row_id_attribute and self.row_id_attribute is not None:\n            to_exclude.append(self.row_id_attribute)\n\n        result = []\n        offset = 0\n        # this function assumes that everything in to_exclude will\n        # be 'excluded' from the dataset (hence the offset)\n        for idx in self.features:\n            name = self.features[idx].name\n            if name in to_exclude:\n                offset += 1\n            elif self.features[idx].data_type == data_type:\n                result.append(idx - offset)\n        return result\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Adds the 'dataset' to file elements.\"\"\"\n        file_elements: dict = {}\n        path = None if self.data_file is None else Path(self.data_file).absolute()\n\n        if self._dataset is not None:\n            file_elements[\"dataset\"] = self._dataset\n        elif path is not None and path.exists():\n            with path.open(\"rb\") as fp:\n                file_elements[\"dataset\"] = fp.read()\n\n            try:\n                dataset_utf8 = str(file_elements[\"dataset\"], encoding=\"utf8\")\n                arff.ArffDecoder().decode(dataset_utf8, encode_nominal=True)\n            except arff.ArffException as e:\n                raise ValueError(\"The file you have provided is not a valid arff file.\") from e\n\n        elif self.url is None:\n            raise ValueError(\"No valid url/path to the data file was given.\")\n        return file_elements\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.dataset_id = int(xml_response[\"oml:upload_data_set\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        props = [\n            \"id\",\n            \"name\",\n            \"version\",\n            \"description\",\n            \"format\",\n            \"creator\",\n            \"contributor\",\n            \"collection_date\",\n            \"upload_date\",\n            \"language\",\n            \"licence\",\n            \"url\",\n            \"default_target_attribute\",\n            \"row_id_attribute\",\n            \"ignore_attribute\",\n            \"version_label\",\n            \"citation\",\n            \"tag\",\n            \"visibility\",\n            \"original_data_url\",\n            \"paper_url\",\n            \"update_comment\",\n            \"md5_checksum\",\n        ]\n\n        prop_values = {}\n        for prop in props:\n            content = getattr(self, prop, None)\n            if content is not None:\n                prop_values[\"oml:\" + prop] = content\n\n        return {\n            \"oml:data_set_description\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **prop_values,\n            }\n        }\n</code></pre>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.features","title":"<code>features: dict[int, OpenMLDataFeature]</code>  <code>property</code>","text":"<p>Get the features of this dataset.</p>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Get the dataset numeric id.</p>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.qualities","title":"<code>qualities: dict[str, float] | None</code>  <code>property</code>","text":"<p>Get the qualities of this dataset.</p>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.get_data","title":"<code>get_data(target=None, include_row_id=False, include_ignore_attribute=False, dataset_format='dataframe')</code>","text":"<p>Returns dataset content as dataframes or sparse matrices.</p> <p>Parameters:</p> Name Type Description Default <code>target</code> <code>(string, List[str] or None(default=None))</code> <p>Name of target column to separate from the data. Splitting multiple columns is currently not supported.</p> <code>None</code> <code>include_row_id</code> <code>boolean(default=False)</code> <p>Whether to include row ids in the returned dataset.</p> <code>False</code> <code>include_ignore_attribute</code> <code>boolean(default=False)</code> <p>Whether to include columns that are marked as \"ignore\" on the server in the dataset.</p> <code>False</code> <code>dataset_format</code> <code>string(default='dataframe')</code> <p>The format of returned dataset. If <code>array</code>, the returned dataset will be a NumPy array or a SciPy sparse matrix. Support for <code>array</code> will be removed in 0.15. If <code>dataframe</code>, the returned dataset will be a Pandas DataFrame.</p> <code>'dataframe'</code> <p>Returns:</p> Name Type Description <code>X</code> <code>ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)</code> <p>Dataset</p> <code>y</code> <code>(ndarray or Series, shape(n_samples) or None)</code> <p>Target column</p> <code>categorical_indicator</code> <code>boolean ndarray</code> <p>Mask that indicate categorical features.</p> <code>attribute_names</code> <code>List[str]</code> <p>List of attribute names.</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_data(  # noqa: C901, PLR0912, PLR0915\n    self,\n    target: list[str] | str | None = None,\n    include_row_id: bool = False,  # noqa: FBT001, FBT002\n    include_ignore_attribute: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.csr_matrix,\n    np.ndarray | pd.DataFrame | None,\n    list[bool],\n    list[str],\n]:\n    \"\"\"Returns dataset content as dataframes or sparse matrices.\n\n    Parameters\n    ----------\n    target : string, List[str] or None (default=None)\n        Name of target column to separate from the data.\n        Splitting multiple columns is currently not supported.\n    include_row_id : boolean (default=False)\n        Whether to include row ids in the returned dataset.\n    include_ignore_attribute : boolean (default=False)\n        Whether to include columns that are marked as \"ignore\"\n        on the server in the dataset.\n    dataset_format : string (default='dataframe')\n        The format of returned dataset.\n        If ``array``, the returned dataset will be a NumPy array or a SciPy sparse\n        matrix. Support for ``array`` will be removed in 0.15.\n        If ``dataframe``, the returned dataset will be a Pandas DataFrame.\n\n\n    Returns\n    -------\n    X : ndarray, dataframe, or sparse matrix, shape (n_samples, n_columns)\n        Dataset\n    y : ndarray or pd.Series, shape (n_samples, ) or None\n        Target column\n    categorical_indicator : boolean ndarray\n        Mask that indicate categorical features.\n    attribute_names : List[str]\n        List of attribute names.\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    data, categorical, attribute_names = self._load_data()\n\n    to_exclude = []\n    if not include_row_id and self.row_id_attribute is not None:\n        if isinstance(self.row_id_attribute, str):\n            to_exclude.append(self.row_id_attribute)\n        elif isinstance(self.row_id_attribute, Iterable):\n            to_exclude.extend(self.row_id_attribute)\n\n    if not include_ignore_attribute and self.ignore_attribute is not None:\n        if isinstance(self.ignore_attribute, str):\n            to_exclude.append(self.ignore_attribute)\n        elif isinstance(self.ignore_attribute, Iterable):\n            to_exclude.extend(self.ignore_attribute)\n\n    if len(to_exclude) &gt; 0:\n        logger.info(\"Going to remove the following attributes: %s\" % to_exclude)\n        keep = np.array([column not in to_exclude for column in attribute_names])\n        data = data.loc[:, keep] if isinstance(data, pd.DataFrame) else data[:, keep]\n\n        categorical = [cat for cat, k in zip(categorical, keep) if k]\n        attribute_names = [att for att, k in zip(attribute_names, keep) if k]\n\n    if target is None:\n        data = self._convert_array_format(data, dataset_format, attribute_names)  # type: ignore\n        targets = None\n    else:\n        if isinstance(target, str):\n            target = target.split(\",\") if \",\" in target else [target]\n        targets = np.array([column in target for column in attribute_names])\n        target_names = [column for column in attribute_names if column in target]\n        if np.sum(targets) &gt; 1:\n            raise NotImplementedError(\n                \"Number of requested targets %d is not implemented.\" % np.sum(targets),\n            )\n        target_categorical = [\n            cat for cat, column in zip(categorical, attribute_names) if column in target\n        ]\n        target_dtype = int if target_categorical[0] else float\n\n        if isinstance(data, pd.DataFrame):\n            x = data.iloc[:, ~targets]\n            y = data.iloc[:, targets]\n        else:\n            x = data[:, ~targets]\n            y = data[:, targets].astype(target_dtype)  # type: ignore\n\n        categorical = [cat for cat, t in zip(categorical, targets) if not t]\n        attribute_names = [att for att, k in zip(attribute_names, targets) if not k]\n\n        x = self._convert_array_format(x, dataset_format, attribute_names)  # type: ignore\n        if dataset_format == \"array\" and scipy.sparse.issparse(y):\n            # scikit-learn requires dense representation of targets\n            y = np.asarray(y.todense()).astype(target_dtype)\n            # dense representation of single column sparse arrays become a 2-d array\n            # need to flatten it to a 1-d array for _convert_array_format()\n            y = y.squeeze()\n        y = self._convert_array_format(y, dataset_format, target_names)\n        y = y.astype(target_dtype) if isinstance(y, np.ndarray) else y\n        if len(y.shape) &gt; 1 and y.shape[1] == 1:\n            # single column targets should be 1-d for both `array` and `dataframe` formats\n            assert isinstance(y, (np.ndarray, pd.DataFrame, pd.Series))\n            y = y.squeeze()\n        data, targets = x, y\n\n    return data, targets, categorical, attribute_names  # type: ignore\n</code></pre>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.get_features_by_type","title":"<code>get_features_by_type(data_type, exclude=None, exclude_ignore_attribute=True, exclude_row_id_attribute=True)</code>","text":"<p>Return indices of features of a given type, e.g. all nominal features. Optional parameters to exclude various features by index or ontology.</p> <p>Parameters:</p> Name Type Description Default <code>data_type</code> <code>str</code> <p>The data type to return (e.g., nominal, numeric, date, string)</p> required <code>exclude</code> <code>list(int)</code> <p>List of columns to exclude from the return value</p> <code>None</code> <code>exclude_ignore_attribute</code> <code>bool</code> <p>Whether to exclude the defined ignore attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <code>exclude_row_id_attribute</code> <code>bool</code> <p>Whether to exclude the defined row id attributes (and adapt the return values as if these indices are not present)</p> <code>True</code> <p>Returns:</p> Name Type Description <code>result</code> <code>list</code> <p>a list of indices that have the specified data type</p> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def get_features_by_type(  # noqa: C901\n    self,\n    data_type: str,\n    exclude: list[str] | None = None,\n    exclude_ignore_attribute: bool = True,  # noqa: FBT002, FBT001\n    exclude_row_id_attribute: bool = True,  # noqa: FBT002, FBT001\n) -&gt; list[int]:\n    \"\"\"\n    Return indices of features of a given type, e.g. all nominal features.\n    Optional parameters to exclude various features by index or ontology.\n\n    Parameters\n    ----------\n    data_type : str\n        The data type to return (e.g., nominal, numeric, date, string)\n    exclude : list(int)\n        List of columns to exclude from the return value\n    exclude_ignore_attribute : bool\n        Whether to exclude the defined ignore attributes (and adapt the\n        return values as if these indices are not present)\n    exclude_row_id_attribute : bool\n        Whether to exclude the defined row id attributes (and adapt the\n        return values as if these indices are not present)\n\n    Returns\n    -------\n    result : list\n        a list of indices that have the specified data type\n    \"\"\"\n    if data_type not in OpenMLDataFeature.LEGAL_DATA_TYPES:\n        raise TypeError(\"Illegal feature type requested\")\n    if self.ignore_attribute is not None and not isinstance(self.ignore_attribute, list):\n        raise TypeError(\"ignore_attribute should be a list\")\n    if self.row_id_attribute is not None and not isinstance(self.row_id_attribute, str):\n        raise TypeError(\"row id attribute should be a str\")\n    if exclude is not None and not isinstance(exclude, list):\n        raise TypeError(\"Exclude should be a list\")\n        # assert all(isinstance(elem, str) for elem in exclude),\n        #            \"Exclude should be a list of strings\"\n    to_exclude = []\n    if exclude is not None:\n        to_exclude.extend(exclude)\n    if exclude_ignore_attribute and self.ignore_attribute is not None:\n        to_exclude.extend(self.ignore_attribute)\n    if exclude_row_id_attribute and self.row_id_attribute is not None:\n        to_exclude.append(self.row_id_attribute)\n\n    result = []\n    offset = 0\n    # this function assumes that everything in to_exclude will\n    # be 'excluded' from the dataset (hence the offset)\n    for idx in self.features:\n        name = self.features[idx].name\n        if name in to_exclude:\n            offset += 1\n        elif self.features[idx].data_type == data_type:\n            result.append(idx - offset)\n    return result\n</code></pre>"},{"location":"reference/datasets/dataset/#openml.datasets.dataset.OpenMLDataset.retrieve_class_labels","title":"<code>retrieve_class_labels(target_name='class')</code>","text":"<p>Reads the datasets arff to determine the class-labels.</p> <p>If the task has no class labels (for example a regression problem) it returns None. Necessary because the data returned by get_data only contains the indices of the classes, while OpenML needs the real classname when uploading the results of a run.</p> <p>Parameters:</p> Name Type Description Default <code>target_name</code> <code>str</code> <p>Name of the target attribute</p> <code>'class'</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/dataset.py</code> <pre><code>def retrieve_class_labels(self, target_name: str = \"class\") -&gt; None | list[str]:\n    \"\"\"Reads the datasets arff to determine the class-labels.\n\n    If the task has no class labels (for example a regression problem)\n    it returns None. Necessary because the data returned by get_data\n    only contains the indices of the classes, while OpenML needs the real\n    classname when uploading the results of a run.\n\n    Parameters\n    ----------\n    target_name : str\n        Name of the target attribute\n\n    Returns\n    -------\n    list\n    \"\"\"\n    for feature in self.features.values():\n        if feature.name == target_name:\n            if feature.data_type == \"nominal\":\n                return feature.nominal_values\n\n            if feature.data_type == \"string\":\n                # Rel.: #1311\n                # The target is invalid for a classification task if the feature type is string\n                # and not nominal. For such miss-configured tasks, we silently fix it here as\n                # we can safely interpreter string as nominal.\n                df, *_ = self.get_data()\n                return list(df[feature.name].unique())\n\n    return None\n</code></pre>"},{"location":"reference/datasets/functions/","title":"functions","text":""},{"location":"reference/datasets/functions/#openml.datasets.functions.attributes_arff_from_df","title":"<code>attributes_arff_from_df(df)</code>","text":"<p>Describe attributes of the dataframe according to ARFF specification.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>(DataFrame, shape(n_samples, n_features))</code> <p>The dataframe containing the data set.</p> required <p>Returns:</p> Name Type Description <code>attributes_arff</code> <code>list[str]</code> <p>The data set attributes as required by the ARFF format.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def attributes_arff_from_df(df: pd.DataFrame) -&gt; list[tuple[str, list[str] | str]]:\n    \"\"\"Describe attributes of the dataframe according to ARFF specification.\n\n    Parameters\n    ----------\n    df : DataFrame, shape (n_samples, n_features)\n        The dataframe containing the data set.\n\n    Returns\n    -------\n    attributes_arff : list[str]\n        The data set attributes as required by the ARFF format.\n    \"\"\"\n    PD_DTYPES_TO_ARFF_DTYPE = {\"integer\": \"INTEGER\", \"floating\": \"REAL\", \"string\": \"STRING\"}\n    attributes_arff: list[tuple[str, list[str] | str]] = []\n\n    if not all(isinstance(column_name, str) for column_name in df.columns):\n        logger.warning(\"Converting non-str column names to str.\")\n        df.columns = [str(column_name) for column_name in df.columns]\n\n    for column_name in df:\n        # skipna=True does not infer properly the dtype. The NA values are\n        # dropped before the inference instead.\n        column_dtype = pd.api.types.infer_dtype(df[column_name].dropna(), skipna=False)\n\n        if column_dtype == \"categorical\":\n            # for categorical feature, arff expects a list string. However, a\n            # categorical column can contain mixed type and should therefore\n            # raise an error asking to convert all entries to string.\n            categories = df[column_name].cat.categories\n            categories_dtype = pd.api.types.infer_dtype(categories)\n            if categories_dtype not in (\"string\", \"unicode\"):\n                raise ValueError(\n                    f\"The column '{column_name}' of the dataframe is of \"\n                    \"'category' dtype. Therefore, all values in \"\n                    \"this columns should be string. Please \"\n                    \"convert the entries which are not string. \"\n                    f\"Got {categories_dtype} dtype in this column.\",\n                )\n            attributes_arff.append((column_name, categories.tolist()))\n        elif column_dtype == \"boolean\":\n            # boolean are encoded as categorical.\n            attributes_arff.append((column_name, [\"True\", \"False\"]))\n        elif column_dtype in PD_DTYPES_TO_ARFF_DTYPE:\n            attributes_arff.append((column_name, PD_DTYPES_TO_ARFF_DTYPE[column_dtype]))\n        else:\n            raise ValueError(\n                f\"The dtype '{column_dtype}' of the column '{column_name}' is not \"\n                \"currently supported by liac-arff. Supported \"\n                \"dtypes are categorical, string, integer, \"\n                \"floating, and boolean.\",\n            )\n    return attributes_arff\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.check_datasets_active","title":"<code>check_datasets_active(dataset_ids, raise_error_if_not_exist=True)</code>","text":"<p>Check if the dataset ids provided are active.</p> <p>Raises an error if a dataset_id in the given list of dataset_ids does not exist on the server and <code>raise_error_if_not_exist</code> is set to True (default).</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>List[int]</code> <p>A list of integers representing dataset ids.</p> required <code>raise_error_if_not_exist</code> <code>bool(default=True)</code> <p>Flag that if activated can raise an error, if one or more of the given dataset ids do not exist on the server.</p> <code>True</code> <p>Returns:</p> Type Description <code>dict</code> <p>A dictionary with items {did: bool}</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def check_datasets_active(\n    dataset_ids: list[int],\n    raise_error_if_not_exist: bool = True,  # noqa: FBT001, FBT002\n) -&gt; dict[int, bool]:\n    \"\"\"\n    Check if the dataset ids provided are active.\n\n    Raises an error if a dataset_id in the given list\n    of dataset_ids does not exist on the server and\n    `raise_error_if_not_exist` is set to True (default).\n\n    Parameters\n    ----------\n    dataset_ids : List[int]\n        A list of integers representing dataset ids.\n    raise_error_if_not_exist : bool (default=True)\n        Flag that if activated can raise an error, if one or more of the\n        given dataset ids do not exist on the server.\n\n    Returns\n    -------\n    dict\n        A dictionary with items {did: bool}\n    \"\"\"\n    datasets = list_datasets(status=\"all\", data_id=dataset_ids, output_format=\"dataframe\")\n    missing = set(dataset_ids) - set(datasets.get(\"did\", []))\n    if raise_error_if_not_exist and missing:\n        missing_str = \", \".join(str(did) for did in missing)\n        raise ValueError(f\"Could not find dataset(s) {missing_str} in OpenML dataset list.\")\n    return dict(datasets[\"status\"] == \"active\")\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.create_dataset","title":"<code>create_dataset(name, description, creator, contributor, collection_date, language, licence, attributes, data, default_target_attribute, ignore_attribute, citation, row_id_attribute=None, original_data_url=None, paper_url=None, update_comment=None, version_label=None)</code>","text":"<p>Create a dataset.</p> <p>This function creates an OpenMLDataset object. The OpenMLDataset object contains information related to the dataset and the actual data file.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> required <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> required <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> required <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> required <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> required <code>licence</code> <code>str</code> <p>License of the data.</p> required <code>attributes</code> <code>list, dict, or 'auto'</code> <p>A list of tuples. Each tuple consists of the attribute name and type. If passing a pandas DataFrame, the attributes can be automatically inferred by passing <code>'auto'</code>. Specific attributes can be manually specified by a passing a dictionary where the key is the name of the attribute and the value is the data type of the attribute.</p> required <code>data</code> <code>(ndarray, list, dataframe, coo_matrix, shape(n_samples, n_features))</code> <p>An array that contains both the attributes and the targets. When providing a dataframe, the attribute names and type can be inferred by passing <code>attributes='auto'</code>. The target feature is indicated as meta-data of the dataset.</p> required <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> required <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes. Can have multiple values, comma separated.</p> required <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> required <code>version_label</code> <code>str</code> <p>Version label provided by user.  Can be a date, hash, or some other type of id.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <code>update_comment</code> <code>str</code> <p>An explanation for when the dataset is uploaded.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>class</code> <code>`openml.OpenMLDataset`</code> <code>Dataset description.</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def create_dataset(  # noqa: C901, PLR0912, PLR0915\n    name: str,\n    description: str | None,\n    creator: str | None,\n    contributor: str | None,\n    collection_date: str | None,\n    language: str | None,\n    licence: str | None,\n    # TODO(eddiebergman): Docstring says `type` but I don't know what this is other than strings\n    # Edit: Found it could also be like [\"True\", \"False\"]\n    attributes: list[tuple[str, str | list[str]]] | dict[str, str | list[str]] | Literal[\"auto\"],\n    data: pd.DataFrame | np.ndarray | scipy.sparse.coo_matrix,\n    # TODO(eddiebergman): Function requires `default_target_attribute` exist but API allows None\n    default_target_attribute: str,\n    ignore_attribute: str | list[str] | None,\n    citation: str,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n    update_comment: str | None = None,\n    version_label: str | None = None,\n) -&gt; OpenMLDataset:\n    \"\"\"Create a dataset.\n\n    This function creates an OpenMLDataset object.\n    The OpenMLDataset object contains information related to the dataset\n    and the actual data file.\n\n    Parameters\n    ----------\n    name : str\n        Name of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    licence : str\n        License of the data.\n    attributes : list, dict, or 'auto'\n        A list of tuples. Each tuple consists of the attribute name and type.\n        If passing a pandas DataFrame, the attributes can be automatically\n        inferred by passing ``'auto'``. Specific attributes can be manually\n        specified by a passing a dictionary where the key is the name of the\n        attribute and the value is the data type of the attribute.\n    data : ndarray, list, dataframe, coo_matrix, shape (n_samples, n_features)\n        An array that contains both the attributes and the targets. When\n        providing a dataframe, the attribute names and type can be inferred by\n        passing ``attributes='auto'``.\n        The target feature is indicated as meta-data of the dataset.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n        Can have multiple values, comma separated.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    version_label : str, optional\n        Version label provided by user.\n         Can be a date, hash, or some other type of id.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n    update_comment : str, optional\n        An explanation for when the dataset is uploaded.\n\n    Returns\n    -------\n    class:`openml.OpenMLDataset`\n    Dataset description.\n    \"\"\"\n    if isinstance(data, pd.DataFrame):\n        # infer the row id from the index of the dataset\n        if row_id_attribute is None:\n            row_id_attribute = data.index.name\n        # When calling data.values, the index will be skipped.\n        # We need to reset the index such that it is part of the data.\n        if data.index.name is not None:\n            data = data.reset_index()\n\n    if attributes == \"auto\" or isinstance(attributes, dict):\n        if not isinstance(data, pd.DataFrame):\n            raise ValueError(\n                \"Automatically inferring attributes requires \"\n                f\"a pandas DataFrame. A {data!r} was given instead.\",\n            )\n        # infer the type of data for each column of the DataFrame\n        attributes_ = attributes_arff_from_df(data)\n        if isinstance(attributes, dict):\n            # override the attributes which was specified by the user\n            for attr_idx in range(len(attributes_)):\n                attr_name = attributes_[attr_idx][0]\n                if attr_name in attributes:\n                    attributes_[attr_idx] = (attr_name, attributes[attr_name])\n    else:\n        attributes_ = attributes\n    ignore_attributes = _expand_parameter(ignore_attribute)\n    _validated_data_attributes(ignore_attributes, attributes_, \"ignore_attribute\")\n\n    default_target_attributes = _expand_parameter(default_target_attribute)\n    _validated_data_attributes(default_target_attributes, attributes_, \"default_target_attribute\")\n\n    if row_id_attribute is not None:\n        is_row_id_an_attribute = any(attr[0] == row_id_attribute for attr in attributes_)\n        if not is_row_id_an_attribute:\n            raise ValueError(\n                \"'row_id_attribute' should be one of the data attribute. \"\n                \" Got '{}' while candidates are {}.\".format(\n                    row_id_attribute,\n                    [attr[0] for attr in attributes_],\n                ),\n            )\n\n    if isinstance(data, pd.DataFrame):\n        if all(isinstance(dtype, pd.SparseDtype) for dtype in data.dtypes):\n            data = data.sparse.to_coo()\n            # liac-arff only support COO matrices with sorted rows\n            row_idx_sorted = np.argsort(data.row)  # type: ignore\n            data.row = data.row[row_idx_sorted]  # type: ignore\n            data.col = data.col[row_idx_sorted]  # type: ignore\n            data.data = data.data[row_idx_sorted]  # type: ignore\n        else:\n            data = data.to_numpy()\n\n    data_format: Literal[\"arff\", \"sparse_arff\"]\n    if isinstance(data, (list, np.ndarray)):\n        if isinstance(data[0], (list, np.ndarray)):\n            data_format = \"arff\"\n        elif isinstance(data[0], dict):\n            data_format = \"sparse_arff\"\n        else:\n            raise ValueError(\n                \"When giving a list or a numpy.ndarray, \"\n                \"they should contain a list/ numpy.ndarray \"\n                \"for dense data or a dictionary for sparse \"\n                f\"data. Got {data[0]!r} instead.\",\n            )\n    elif isinstance(data, coo_matrix):\n        data_format = \"sparse_arff\"\n    else:\n        raise ValueError(\n            \"When giving a list or a numpy.ndarray, \"\n            \"they should contain a list/ numpy.ndarray \"\n            \"for dense data or a dictionary for sparse \"\n            f\"data. Got {data[0]!r} instead.\",\n        )\n\n    arff_object = {\n        \"relation\": name,\n        \"description\": description,\n        \"attributes\": attributes_,\n        \"data\": data,\n    }\n\n    # serializes the ARFF dataset object and returns a string\n    arff_dataset = arff.dumps(arff_object)\n    try:\n        # check if ARFF is valid\n        decoder = arff.ArffDecoder()\n        return_type = arff.COO if data_format == \"sparse_arff\" else arff.DENSE\n        decoder.decode(arff_dataset, encode_nominal=True, return_type=return_type)\n    except arff.ArffException as e:\n        raise ValueError(\n            \"The arguments you have provided do not construct a valid ARFF file\"\n        ) from e\n\n    return OpenMLDataset(\n        name=name,\n        description=description,\n        data_format=data_format,\n        creator=creator,\n        contributor=contributor,\n        collection_date=collection_date,\n        language=language,\n        licence=licence,\n        default_target_attribute=default_target_attribute,\n        row_id_attribute=row_id_attribute,\n        ignore_attribute=ignore_attribute,\n        citation=citation,\n        version_label=version_label,\n        original_data_url=original_data_url,\n        paper_url=paper_url,\n        update_comment=update_comment,\n        dataset=arff_dataset,\n    )\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.data_feature_add_ontology","title":"<code>data_feature_add_ontology(data_id, index, ontology)</code>","text":"<p>An ontology describes the concept that are described in a feature. An ontology is defined by an URL where the information is provided. Adds an ontology (URL) to a given dataset feature (defined by a dataset id and index). The dataset has to exists on OpenML and needs to have been processed by the evaluation engine.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to which the feature belongs</p> required <code>index</code> <code>int</code> <p>index of the feature in dataset (0-based)</p> required <code>ontology</code> <code>str</code> <p>URL to ontology (max. 256 characters)</p> required <p>Returns:</p> Type Description <code>True or throws an OpenML server exception</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def data_feature_add_ontology(data_id: int, index: int, ontology: str) -&gt; bool:\n    \"\"\"\n    An ontology describes the concept that are described in a feature. An\n    ontology is defined by an URL where the information is provided. Adds\n    an ontology (URL) to a given dataset feature (defined by a dataset id\n    and index). The dataset has to exists on OpenML and needs to have been\n    processed by the evaluation engine.\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to which the feature belongs\n    index : int\n        index of the feature in dataset (0-based)\n    ontology : str\n        URL to ontology (max. 256 characters)\n\n    Returns\n    -------\n    True or throws an OpenML server exception\n    \"\"\"\n    upload_data: dict[str, int | str] = {\"data_id\": data_id, \"index\": index, \"ontology\": ontology}\n    openml._api_calls._perform_api_call(\"data/feature/ontology/add\", \"post\", data=upload_data)\n    # an error will be thrown in case the request was unsuccessful\n    return True\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.data_feature_remove_ontology","title":"<code>data_feature_remove_ontology(data_id, index, ontology)</code>","text":"<p>Removes an existing ontology (URL) from a given dataset feature (defined by a dataset id and index). The dataset has to exists on OpenML and needs to have been processed by the evaluation engine. Ontology needs to be attached to the specific fearure.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to which the feature belongs</p> required <code>index</code> <code>int</code> <p>index of the feature in dataset (0-based)</p> required <code>ontology</code> <code>str</code> <p>URL to ontology (max. 256 characters)</p> required <p>Returns:</p> Type Description <code>True or throws an OpenML server exception</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def data_feature_remove_ontology(data_id: int, index: int, ontology: str) -&gt; bool:\n    \"\"\"\n    Removes an existing ontology (URL) from a given dataset feature (defined\n    by a dataset id and index). The dataset has to exists on OpenML and needs\n    to have been processed by the evaluation engine. Ontology needs to be\n    attached to the specific fearure.\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to which the feature belongs\n    index : int\n        index of the feature in dataset (0-based)\n    ontology : str\n        URL to ontology (max. 256 characters)\n\n    Returns\n    -------\n    True or throws an OpenML server exception\n    \"\"\"\n    upload_data: dict[str, int | str] = {\"data_id\": data_id, \"index\": index, \"ontology\": ontology}\n    openml._api_calls._perform_api_call(\"data/feature/ontology/remove\", \"post\", data=upload_data)\n    # an error will be thrown in case the request was unsuccessful\n    return True\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.delete_dataset","title":"<code>delete_dataset(dataset_id)</code>","text":"<p>Delete dataset with id <code>dataset_id</code> from the OpenML server.</p> <p>This can only be done if you are the owner of the dataset and no tasks are attached to the dataset.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int</code> <p>OpenML id of the dataset</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def delete_dataset(dataset_id: int) -&gt; bool:\n    \"\"\"Delete dataset with id `dataset_id` from the OpenML server.\n\n    This can only be done if you are the owner of the dataset and\n    no tasks are attached to the dataset.\n\n    Parameters\n    ----------\n    dataset_id : int\n        OpenML id of the dataset\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"data\", dataset_id)\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.edit_dataset","title":"<code>edit_dataset(data_id, description=None, creator=None, contributor=None, collection_date=None, language=None, default_target_attribute=None, ignore_attribute=None, citation=None, row_id_attribute=None, original_data_url=None, paper_url=None)</code>","text":"<p>Edits an OpenMLDataset.</p> <p>In addition to providing the dataset id of the dataset to edit (through data_id), you must specify a value for at least one of the optional function arguments, i.e. one value for a field to edit.</p> <p>This function allows editing of both non-critical and critical fields. Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.</p> <ul> <li>Editing non-critical data fields is allowed for all authenticated users.</li> <li>Editing critical fields is allowed only for the owner, provided there are no tasks    associated with this dataset.</li> </ul> <p>If dataset has tasks or if the user is not the owner, the only way to edit critical fields is to use fork_dataset followed by edit_dataset.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>ID of the dataset.</p> required <code>description</code> <code>str</code> <p>Description of the dataset.</p> <code>None</code> <code>creator</code> <code>str</code> <p>The person who created the dataset.</p> <code>None</code> <code>contributor</code> <code>str</code> <p>People who contributed to the current version of the dataset.</p> <code>None</code> <code>collection_date</code> <code>str</code> <p>The date the data was originally collected, given by the uploader.</p> <code>None</code> <code>language</code> <code>str</code> <p>Language in which the data is represented. Starts with 1 upper case letter, rest lower case, e.g. 'English'.</p> <code>None</code> <code>default_target_attribute</code> <code>str</code> <p>The default target attribute, if it exists. Can have multiple values, comma separated.</p> <code>None</code> <code>ignore_attribute</code> <code>str | list</code> <p>Attributes that should be excluded in modelling, such as identifiers and indexes.</p> <code>None</code> <code>citation</code> <code>str</code> <p>Reference(s) that should be cited when building on this data.</p> <code>None</code> <code>row_id_attribute</code> <code>str</code> <p>The attribute that represents the row-id column, if present in the dataset. If <code>data</code> is a dataframe and <code>row_id_attribute</code> is not specified, the index of the dataframe will be used as the <code>row_id_attribute</code>. If the name of the index is <code>None</code>, it will be discarded.</p> <p>.. versionadded: 0.8     Inference of <code>row_id_attribute</code> from a dataframe.</p> <code>None</code> <code>original_data_url</code> <code>str</code> <p>For derived data, the url to the original dataset.</p> <code>None</code> <code>paper_url</code> <code>str</code> <p>Link to a paper describing the dataset.</p> <code>None</code> <p>Returns:</p> Type Description <code>Dataset id</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def edit_dataset(\n    data_id: int,\n    description: str | None = None,\n    creator: str | None = None,\n    contributor: str | None = None,\n    collection_date: str | None = None,\n    language: str | None = None,\n    default_target_attribute: str | None = None,\n    ignore_attribute: str | list[str] | None = None,\n    citation: str | None = None,\n    row_id_attribute: str | None = None,\n    original_data_url: str | None = None,\n    paper_url: str | None = None,\n) -&gt; int:\n    \"\"\"Edits an OpenMLDataset.\n\n    In addition to providing the dataset id of the dataset to edit (through data_id),\n    you must specify a value for at least one of the optional function arguments,\n    i.e. one value for a field to edit.\n\n    This function allows editing of both non-critical and critical fields.\n    Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.\n\n     - Editing non-critical data fields is allowed for all authenticated users.\n     - Editing critical fields is allowed only for the owner, provided there are no tasks\n       associated with this dataset.\n\n    If dataset has tasks or if the user is not the owner, the only way\n    to edit critical fields is to use fork_dataset followed by edit_dataset.\n\n    Parameters\n    ----------\n    data_id : int\n        ID of the dataset.\n    description : str\n        Description of the dataset.\n    creator : str\n        The person who created the dataset.\n    contributor : str\n        People who contributed to the current version of the dataset.\n    collection_date : str\n        The date the data was originally collected, given by the uploader.\n    language : str\n        Language in which the data is represented.\n        Starts with 1 upper case letter, rest lower case, e.g. 'English'.\n    default_target_attribute : str\n        The default target attribute, if it exists.\n        Can have multiple values, comma separated.\n    ignore_attribute : str | list\n        Attributes that should be excluded in modelling,\n        such as identifiers and indexes.\n    citation : str\n        Reference(s) that should be cited when building on this data.\n    row_id_attribute : str, optional\n        The attribute that represents the row-id column, if present in the\n        dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not\n        specified, the index of the dataframe will be used as the\n        ``row_id_attribute``. If the name of the index is ``None``, it will\n        be discarded.\n\n        .. versionadded: 0.8\n            Inference of ``row_id_attribute`` from a dataframe.\n    original_data_url : str, optional\n        For derived data, the url to the original dataset.\n    paper_url : str, optional\n        Link to a paper describing the dataset.\n\n    Returns\n    -------\n    Dataset id\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n\n    # compose data edit parameters as xml\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    xml = OrderedDict()  # type: 'OrderedDict[str, OrderedDict]'\n    xml[\"oml:data_edit_parameters\"] = OrderedDict()\n    xml[\"oml:data_edit_parameters\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n    xml[\"oml:data_edit_parameters\"][\"oml:description\"] = description\n    xml[\"oml:data_edit_parameters\"][\"oml:creator\"] = creator\n    xml[\"oml:data_edit_parameters\"][\"oml:contributor\"] = contributor\n    xml[\"oml:data_edit_parameters\"][\"oml:collection_date\"] = collection_date\n    xml[\"oml:data_edit_parameters\"][\"oml:language\"] = language\n    xml[\"oml:data_edit_parameters\"][\"oml:default_target_attribute\"] = default_target_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:row_id_attribute\"] = row_id_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:ignore_attribute\"] = ignore_attribute\n    xml[\"oml:data_edit_parameters\"][\"oml:citation\"] = citation\n    xml[\"oml:data_edit_parameters\"][\"oml:original_data_url\"] = original_data_url\n    xml[\"oml:data_edit_parameters\"][\"oml:paper_url\"] = paper_url\n\n    # delete None inputs\n    for k in list(xml[\"oml:data_edit_parameters\"]):\n        if not xml[\"oml:data_edit_parameters\"][k]:\n            del xml[\"oml:data_edit_parameters\"][k]\n\n    file_elements = {\n        \"edit_parameters\": (\"description.xml\", xmltodict.unparse(xml)),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        \"data/edit\",\n        \"post\",\n        data=form_data,\n        file_elements=file_elements,\n    )\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_edit\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.fork_dataset","title":"<code>fork_dataset(data_id)</code>","text":"<p>Creates a new dataset version, with the authenticated user as the new owner.  The forked dataset can have distinct dataset meta-data,  but the actual data itself is shared with the original version.</p> <p>This API is intended for use when a user is unable to edit the critical fields of a dataset  through the edit_dataset API.  (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)</p> <p>Specifically, this happens when the user is:         1. Not the owner of the dataset.         2. User is the owner of the dataset, but the dataset has tasks.</p> <p>In these two cases the only way to edit critical fields is:         1. STEP 1: Fork the dataset using fork_dataset API         2. STEP 2: Call edit_dataset API on the forked version.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>id of the dataset to be forked</p> required <p>Returns:</p> Type Description <code>Dataset id of the forked dataset</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def fork_dataset(data_id: int) -&gt; int:\n    \"\"\"\n     Creates a new dataset version, with the authenticated user as the new owner.\n     The forked dataset can have distinct dataset meta-data,\n     but the actual data itself is shared with the original version.\n\n     This API is intended for use when a user is unable to edit the critical fields of a dataset\n     through the edit_dataset API.\n     (Critical fields are default_target_attribute, ignore_attribute, row_id_attribute.)\n\n     Specifically, this happens when the user is:\n            1. Not the owner of the dataset.\n            2. User is the owner of the dataset, but the dataset has tasks.\n\n     In these two cases the only way to edit critical fields is:\n            1. STEP 1: Fork the dataset using fork_dataset API\n            2. STEP 2: Call edit_dataset API on the forked version.\n\n\n    Parameters\n    ----------\n    data_id : int\n        id of the dataset to be forked\n\n    Returns\n    -------\n    Dataset id of the forked dataset\n\n    \"\"\"\n    if not isinstance(data_id, int):\n        raise TypeError(f\"`data_id` must be of type `int`, not {type(data_id)}.\")\n    # compose data fork parameters\n    form_data = {\"data_id\": data_id}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"data/fork\", \"post\", data=form_data)\n    result = xmltodict.parse(result_xml)\n    data_id = result[\"oml:data_fork\"][\"oml:id\"]\n    return int(data_id)\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.get_dataset","title":"<code>get_dataset(dataset_id, download_data=None, version=None, error_if_multiple=False, cache_format='pickle', download_qualities=None, download_features_meta_data=None, download_all_files=False, force_refresh_cache=False)</code>","text":"<p>Download the OpenML dataset representation, optionally also download actual data file.</p> <p>This function is by default NOT thread/multiprocessing safe, as this function uses caching. A check will be performed to determine if the information has previously been downloaded to a cache, and if so be loaded from disk instead of retrieved from the server.</p> <p>To make this function thread safe, you can install the python package <code>oslo.concurrency</code>. If <code>oslo.concurrency</code> is installed <code>get_dataset</code> becomes thread safe.</p> <p>Alternatively, to make this function thread/multiprocessing safe initialize the cache first by calling <code>get_dataset(args)</code> once before calling <code>get_dataset(args)</code> many times in parallel. This will initialize the cache and later calls will use the cache in a thread/multiprocessing safe way.</p> <p>If dataset is retrieved by name, a version may be specified. If no version is specified and multiple versions of the dataset exist, the earliest version of the dataset that is still active will be returned. If no version is specified, multiple versions of the dataset exist and <code>exception_if_multiple</code> is set to <code>True</code>, this function will raise an exception.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_id</code> <code>int or str</code> <p>Dataset ID of the dataset to download</p> required <code>download_data</code> <code>bool(default=True)</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>None</code> <code>version</code> <code>(int, optional(default=None))</code> <p>Specifies the version if <code>dataset_id</code> is specified by name. If no version is specified, retrieve the least recent still active version.</p> <code>None</code> <code>error_if_multiple</code> <code>bool(default=False)</code> <p>If <code>True</code> raise an error if multiple datasets are found with matching criteria.</p> <code>False</code> <code>cache_format</code> <code>str(default='pickle') in {'pickle', 'feather'}</code> <p>Format for caching the dataset - may be feather or pickle Note that the default 'pickle' option may load slower than feather when no.of.rows is very high.</p> <code>'pickle'</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description. If True, download and cache the qualities file. If False, create the OpenMLDataset without qualities metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(qualities=True)</code> method.</p> <code>None</code> <code>download_features_meta_data</code> <code>bool(default=True)</code> <p>Option to download 'features' meta-data in addition to the minimal dataset description. If True, download and cache the features file. If False, create the OpenMLDataset without features metadata. The data may later be added to the OpenMLDataset through the <code>OpenMLDataset.load_metadata(features=True)</code> method.</p> <code>None</code> <code>download_all_files</code> <code>bool</code> <p>EXPERIMENTAL. Download all files related to the dataset that reside on the server. Useful for datasets which refer to auxiliary files (e.g., meta-album).</p> <code>False</code> <code>force_refresh_cache</code> <code>bool(default=False)</code> <p>Force the cache to refreshed by deleting the cache directory and re-downloading the data. Note, if <code>force_refresh_cache</code> is True, <code>get_dataset</code> is NOT thread/multiprocessing safe, because this creates a race condition to creating and deleting the cache; as in general with the cache.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>dataset</code> <code>:class:`openml.OpenMLDataset`</code> <p>The downloaded dataset.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_dataset(  # noqa: C901, PLR0912\n    dataset_id: int | str,\n    download_data: bool | None = None,  # Optional for deprecation warning; later again only bool\n    version: int | None = None,\n    error_if_multiple: bool = False,  # noqa: FBT002, FBT001\n    cache_format: Literal[\"pickle\", \"feather\"] = \"pickle\",\n    download_qualities: bool | None = None,  # Same as above\n    download_features_meta_data: bool | None = None,  # Same as above\n    download_all_files: bool = False,  # noqa: FBT002, FBT001\n    force_refresh_cache: bool = False,  # noqa: FBT001, FBT002\n) -&gt; OpenMLDataset:\n    \"\"\"Download the OpenML dataset representation, optionally also download actual data file.\n\n    This function is by default NOT thread/multiprocessing safe, as this function uses caching.\n    A check will be performed to determine if the information has previously been downloaded to a\n    cache, and if so be loaded from disk instead of retrieved from the server.\n\n    To make this function thread safe, you can install the python package ``oslo.concurrency``.\n    If ``oslo.concurrency`` is installed `get_dataset` becomes thread safe.\n\n    Alternatively, to make this function thread/multiprocessing safe initialize the cache first by\n    calling `get_dataset(args)` once before calling `get_dataset(args)` many times in parallel.\n    This will initialize the cache and later calls will use the cache in a thread/multiprocessing\n    safe way.\n\n    If dataset is retrieved by name, a version may be specified.\n    If no version is specified and multiple versions of the dataset exist,\n    the earliest version of the dataset that is still active will be returned.\n    If no version is specified, multiple versions of the dataset exist and\n    ``exception_if_multiple`` is set to ``True``, this function will raise an exception.\n\n    Parameters\n    ----------\n    dataset_id : int or str\n        Dataset ID of the dataset to download\n    download_data : bool (default=True)\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    version : int, optional (default=None)\n        Specifies the version if `dataset_id` is specified by name.\n        If no version is specified, retrieve the least recent still active version.\n    error_if_multiple : bool (default=False)\n        If ``True`` raise an error if multiple datasets are found with matching criteria.\n    cache_format : str (default='pickle') in {'pickle', 'feather'}\n        Format for caching the dataset - may be feather or pickle\n        Note that the default 'pickle' option may load slower than feather when\n        no.of.rows is very high.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n        If True, download and cache the qualities file.\n        If False, create the OpenMLDataset without qualities metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(qualities=True)` method.\n    download_features_meta_data : bool (default=True)\n        Option to download 'features' meta-data in addition to the minimal dataset description.\n        If True, download and cache the features file.\n        If False, create the OpenMLDataset without features metadata. The data may later be added\n        to the OpenMLDataset through the `OpenMLDataset.load_metadata(features=True)` method.\n    download_all_files: bool (default=False)\n        EXPERIMENTAL. Download all files related to the dataset that reside on the server.\n        Useful for datasets which refer to auxiliary files (e.g., meta-album).\n    force_refresh_cache : bool (default=False)\n        Force the cache to refreshed by deleting the cache directory and re-downloading the data.\n        Note, if `force_refresh_cache` is True, `get_dataset` is NOT thread/multiprocessing safe,\n        because this creates a race condition to creating and deleting the cache; as in general with\n        the cache.\n\n    Returns\n    -------\n    dataset : :class:`openml.OpenMLDataset`\n        The downloaded dataset.\n    \"\"\"\n    # TODO(0.15): Remove the deprecation warning and make the default False; adjust types above\n    #   and documentation. Also remove None-to-True-cases below\n    if any(\n        download_flag is None\n        for download_flag in [download_data, download_qualities, download_features_meta_data]\n    ):\n        warnings.warn(\n            \"Starting from Version 0.15 `download_data`, `download_qualities`, and `download_featu\"\n            \"res_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy \"\n            \"loading. To disable this message until version 0.15 explicitly set `download_data`, \"\n            \"`download_qualities`, and `download_features_meta_data` to a bool while calling \"\n            \"`get_dataset`.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    download_data = True if download_data is None else download_data\n    download_qualities = True if download_qualities is None else download_qualities\n    download_features_meta_data = (\n        True if download_features_meta_data is None else download_features_meta_data\n    )\n\n    if download_all_files:\n        warnings.warn(\n            \"``download_all_files`` is experimental and is likely to break with new releases.\",\n            FutureWarning,\n            stacklevel=2,\n        )\n\n    if cache_format not in [\"feather\", \"pickle\"]:\n        raise ValueError(\n            \"cache_format must be one of 'feather' or 'pickle. \"\n            f\"Invalid format specified: {cache_format}\",\n        )\n\n    if isinstance(dataset_id, str):\n        try:\n            dataset_id = int(dataset_id)\n        except ValueError:\n            dataset_id = _name_to_id(dataset_id, version, error_if_multiple)  # type: ignore\n    elif not isinstance(dataset_id, int):\n        raise TypeError(\n            f\"`dataset_id` must be one of `str` or `int`, not {type(dataset_id)}.\",\n        )\n\n    if force_refresh_cache:\n        did_cache_dir = _get_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, dataset_id)\n        if did_cache_dir.exists():\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    did_cache_dir = _create_cache_directory_for_id(\n        DATASETS_CACHE_DIR_NAME,\n        dataset_id,\n    )\n\n    remove_dataset_cache = True\n    try:\n        description = _get_dataset_description(did_cache_dir, dataset_id)\n        features_file = None\n        qualities_file = None\n\n        if download_features_meta_data:\n            features_file = _get_dataset_features_file(did_cache_dir, dataset_id)\n        if download_qualities:\n            qualities_file = _get_dataset_qualities_file(did_cache_dir, dataset_id)\n\n        arff_file = _get_dataset_arff(description) if download_data else None\n        if \"oml:parquet_url\" in description and download_data:\n            try:\n                parquet_file = _get_dataset_parquet(\n                    description,\n                    download_all_files=download_all_files,\n                )\n            except urllib3.exceptions.MaxRetryError:\n                parquet_file = None\n            if parquet_file is None and arff_file:\n                logger.warning(\"Failed to download parquet, fallback on ARFF.\")\n        else:\n            parquet_file = None\n        remove_dataset_cache = False\n    except OpenMLServerException as e:\n        # if there was an exception\n        # check if the user had access to the dataset\n        if e.code == NO_ACCESS_GRANTED_ERRCODE:\n            raise OpenMLPrivateDatasetError(e.message) from None\n\n        raise e\n    finally:\n        if remove_dataset_cache:\n            _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir)\n\n    return _create_dataset_from_description(\n        description,\n        features_file,\n        qualities_file,\n        arff_file,\n        parquet_file,\n        cache_format,\n    )\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.get_datasets","title":"<code>get_datasets(dataset_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download datasets.</p> <p>This function iterates :meth:<code>openml.datasets.get_dataset</code>.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_ids</code> <code>iterable</code> <p>Integers or strings representing dataset ids or dataset names. If dataset names are specified, the least recent still active dataset version is returned.</p> required <code>download_data</code> <code>bool</code> <p>If True, also download the data file. Beware that some datasets are large and it might make the operation noticeably slower. Metadata is also still retrieved. If False, create the OpenMLDataset and only populate it with the metadata. The data may later be retrieved through the <code>OpenMLDataset.get_data</code> method.</p> <code>True</code> <code>download_qualities</code> <code>(bool, optional(default=True))</code> <p>If True, also download qualities.xml file. If False it skip the qualities.xml.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>list of datasets</code> <p>A list of dataset objects.</p> Source code in <code>openml/datasets/functions.py</code> <pre><code>def get_datasets(\n    dataset_ids: list[str | int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLDataset]:\n    \"\"\"Download datasets.\n\n    This function iterates :meth:`openml.datasets.get_dataset`.\n\n    Parameters\n    ----------\n    dataset_ids : iterable\n        Integers or strings representing dataset ids or dataset names.\n        If dataset names are specified, the least recent still active dataset version is returned.\n    download_data : bool, optional\n        If True, also download the data file. Beware that some datasets are large and it might\n        make the operation noticeably slower. Metadata is also still retrieved.\n        If False, create the OpenMLDataset and only populate it with the metadata.\n        The data may later be retrieved through the `OpenMLDataset.get_data` method.\n    download_qualities : bool, optional (default=True)\n        If True, also download qualities.xml file. If False it skip the qualities.xml.\n\n    Returns\n    -------\n    datasets : list of datasets\n        A list of dataset objects.\n    \"\"\"\n    datasets = []\n    for dataset_id in dataset_ids:\n        datasets.append(\n            get_dataset(dataset_id, download_data, download_qualities=download_qualities),\n        )\n    return datasets\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.list_datasets","title":"<code>list_datasets(data_id=None, offset=None, size=None, status=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None, offset: int | None, size: int | None, status: str | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_datasets(data_id: list[int] | None = ..., offset: int | None = ..., size: int | None = ..., status: str | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all dataset which are on OpenML. Supports large amount of results.</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>list</code> <p>A list of data ids, to specify which datasets should be listed</p> <code>None</code> <code>offset</code> <code>int</code> <p>The number of datasets to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of datasets to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated}. By default active datasets are returned, but also datasets from another status can be requested.</p> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>output_format</code> <code>Literal['dataframe', 'dict']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators (keys in the dict): data_name, data_version, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from dataset ID to dict.</p> <p>Every dataset is represented by a dictionary containing the following information: - dataset id - name - format - status If qualities are calculated for the dataset, some of these are also returned.</p> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>dataset id</li> <li>name</li> <li>format</li> <li>status If qualities are calculated for the dataset, some of these are also included as columns.</li> </ul> </li> </ul> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_datasets(\n    data_id: list[int] | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dataframe\", \"dict\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all dataset which are on OpenML.\n    Supports large amount of results.\n\n    Parameters\n    ----------\n    data_id : list, optional\n        A list of data ids, to specify which datasets should be\n        listed\n    offset : int, optional\n        The number of datasets to skip, starting from the first.\n    size : int, optional\n        The maximum number of datasets to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated}. By\n        default active datasets are returned, but also datasets\n        from another status can be requested.\n    tag : str, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs : dict, optional\n        Legal filter operators (keys in the dict):\n        data_name, data_version, number_instances,\n        number_features, number_classes, number_missing_values.\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from dataset ID to dict.\n\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - dataset id\n            - name\n            - format\n            - status\n            If qualities are calculated for the dataset, some of\n            these are also included as columns.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        data_id=data_id,\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_datasets,\n        offset=offset,\n        size=size,\n        status=status,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.list_qualities","title":"<code>list_qualities()</code>","text":"<p>Return list of data qualities available.</p> <p>The function performs an API call to retrieve the entire list of data qualities that are computed on the datasets uploaded.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/datasets/functions.py</code> <pre><code>def list_qualities() -&gt; list[str]:\n    \"\"\"Return list of data qualities available.\n\n    The function performs an API call to retrieve the entire list of\n    data qualities that are computed on the datasets uploaded.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    api_call = \"data/qualities/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:quality\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:data_qualities_list\" not in qualities:\n        raise ValueError('Error in return XML, does not contain \"oml:data_qualities_list\"')\n\n    if not isinstance(qualities[\"oml:data_qualities_list\"][\"oml:quality\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:quality\" as a list')\n\n    return qualities[\"oml:data_qualities_list\"][\"oml:quality\"]\n</code></pre>"},{"location":"reference/datasets/functions/#openml.datasets.functions.status_update","title":"<code>status_update(data_id, status)</code>","text":"<p>Updates the status of a dataset to either 'active' or 'deactivated'. Please see the OpenML API documentation for a description of the status and all legal status transitions: https://docs.openml.org/#dataset-status</p> <p>Parameters:</p> Name Type Description Default <code>data_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/datasets/functions.py</code> <pre><code>def status_update(data_id: int, status: Literal[\"active\", \"deactivated\"]) -&gt; None:\n    \"\"\"\n    Updates the status of a dataset to either 'active' or 'deactivated'.\n    Please see the OpenML API documentation for a description of the status\n    and all legal status transitions:\n    https://docs.openml.org/#dataset-status\n\n    Parameters\n    ----------\n    data_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(f\"Illegal status value. Legal values: {legal_status}\")\n\n    data: openml._api_calls.DATA_TYPE = {\"data_id\": data_id, \"status\": status}\n    result_xml = openml._api_calls._perform_api_call(\"data/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_data_id = result[\"oml:data_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:data_status_update\"][\"oml:status\"]\n    if status != server_status or int(data_id) != int(server_data_id):\n        # This should never happen\n        raise ValueError(\"Data id/status does not collide\")\n</code></pre>"},{"location":"reference/evaluations/","title":"evaluations","text":""},{"location":"reference/evaluations/#openml.evaluations.OpenMLEvaluation","title":"<code>OpenMLEvaluation</code>","text":"<p>Contains all meta-information about a run / evaluation combination, according to the evaluation/list function</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Refers to the run.</p> required <code>task_id</code> <code>int</code> <p>Refers to the task.</p> required <code>setup_id</code> <code>int</code> <p>Refers to the setup.</p> required <code>flow_id</code> <code>int</code> <p>Refers to the flow.</p> required <code>flow_name</code> <code>str</code> <p>Name of the referred flow.</p> required <code>data_id</code> <code>int</code> <p>Refers to the dataset.</p> required <code>data_name</code> <code>str</code> <p>The name of the dataset.</p> required <code>function</code> <code>str</code> <p>The evaluation metric of this item (e.g., accuracy).</p> required <code>upload_time</code> <code>str</code> <p>The time of evaluation.</p> required <code>uploader</code> <code>int</code> <p>Uploader ID (user ID)</p> required <code>upload_name</code> <code>str</code> <p>Name of the uploader of this evaluation</p> required <code>value</code> <code>float</code> <p>The value (score) of this evaluation.</p> required <code>values</code> <code>List[float]</code> <p>The values (scores) per repeat and fold (if requested)</p> required <code>array_data</code> <code>str</code> <p>list of information per class. (e.g., in case of precision, auroc, recall)</p> <code>None</code> Source code in <code>openml/evaluations/evaluation.py</code> <pre><code>class OpenMLEvaluation:\n    \"\"\"\n    Contains all meta-information about a run / evaluation combination,\n    according to the evaluation/list function\n\n    Parameters\n    ----------\n    run_id : int\n        Refers to the run.\n    task_id : int\n        Refers to the task.\n    setup_id : int\n        Refers to the setup.\n    flow_id : int\n        Refers to the flow.\n    flow_name : str\n        Name of the referred flow.\n    data_id : int\n        Refers to the dataset.\n    data_name : str\n        The name of the dataset.\n    function : str\n        The evaluation metric of this item (e.g., accuracy).\n    upload_time : str\n        The time of evaluation.\n    uploader: int\n        Uploader ID (user ID)\n    upload_name : str\n        Name of the uploader of this evaluation\n    value : float\n        The value (score) of this evaluation.\n    values : List[float]\n        The values (scores) per repeat and fold (if requested)\n    array_data : str\n        list of information per class.\n        (e.g., in case of precision, auroc, recall)\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        run_id: int,\n        task_id: int,\n        setup_id: int,\n        flow_id: int,\n        flow_name: str,\n        data_id: int,\n        data_name: str,\n        function: str,\n        upload_time: str,\n        uploader: int,\n        uploader_name: str,\n        value: float | None,\n        values: list[float] | None,\n        array_data: str | None = None,\n    ):\n        self.run_id = run_id\n        self.task_id = task_id\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.data_id = data_id\n        self.data_name = data_name\n        self.function = function\n        self.upload_time = upload_time\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.value = value\n        self.values = values\n        self.array_data = array_data\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Evaluation\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Upload Date\": self.upload_time,\n            \"Run ID\": self.run_id,\n            \"OpenML Run URL\": openml.runs.OpenMLRun.url_for_id(self.run_id),\n            \"Task ID\": self.task_id,\n            \"OpenML Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"OpenML Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Setup ID\": self.setup_id,\n            \"Data ID\": self.data_id,\n            \"Data Name\": self.data_name,\n            \"OpenML Data URL\": openml.datasets.OpenMLDataset.url_for_id(self.data_id),\n            \"Metric Used\": self.function,\n            \"Result\": self.value,\n        }\n\n        order = [\n            \"Uploader Date\",\n            \"Run ID\",\n            \"OpenML Run URL\",\n            \"Task ID\",\n            \"OpenML Task URL\" \"Flow ID\",\n            \"OpenML Flow URL\",\n            \"Setup ID\",\n            \"Data ID\",\n            \"Data Name\",\n            \"OpenML Data URL\",\n            \"Metric Used\",\n            \"Result\",\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/evaluations/#openml.evaluations.list_evaluation_measures","title":"<code>list_evaluation_measures()</code>","text":"<p>Return list of evaluation measures available.</p> <p>The function performs an API call to retrieve the entire list of evaluation measures that are available.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluation_measures() -&gt; list[str]:\n    \"\"\"Return list of evaluation measures available.\n\n    The function performs an API call to retrieve the entire list of\n    evaluation measures that are available.\n\n    Returns\n    -------\n    list\n\n    \"\"\"\n    api_call = \"evaluationmeasure/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:measures\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:evaluation_measures\" not in qualities:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:evaluation_measures\"')\n    if not isinstance(qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:measure\" as a list')\n    return qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"]\n</code></pre>"},{"location":"reference/evaluations/#openml.evaluations.list_evaluations","title":"<code>list_evaluations(function, offset=None, size=10000, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, study=None, per_fold=None, sort_order=None, output_format='object')</code>","text":"<pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dict', 'object'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dataframe'] = ...) -&gt; pd.DataFrame\n</code></pre> <p>List all run-evaluation pairs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of runs to show. If set to <code>None</code>, it returns all the results.</p> <code>10000</code> <code>tasks</code> <code>list[int, str]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list[str | int] | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int, str]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[str | int] | None</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int, str]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'object' the output is a dict of OpenMLEvaluation objects - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations(\n    function: str,\n    offset: int | None = None,\n    size: int | None = 10000,\n    tasks: list[str | int] | None = None,\n    setups: list[str | int] | None = None,\n    flows: list[str | int] | None = None,\n    runs: list[str | int] | None = None,\n    uploaders: list[str | int] | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, default 10000\n        The maximum number of runs to show.\n        If set to ``None``, it returns all the results.\n\n    tasks : list[int,str], optional\n        the list of task IDs\n    setups: list[int,str], optional\n        the list of setup IDs\n    flows : list[int,str], optional\n        the list of flow IDs\n    runs :list[int,str], optional\n        the list of run IDs\n    uploaders : list[int,str], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n\n    study : int, optional\n\n    per_fold : bool, optional\n\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'object' the output is a dict of OpenMLEvaluation objects\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. Only 'object', 'dataframe', or 'dict' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    per_fold_str = None\n    if per_fold is not None:\n        per_fold_str = str(per_fold).lower()\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_evaluations,\n        function=function,\n        offset=offset,\n        size=size,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        runs=runs,\n        uploaders=uploaders,\n        tag=tag,\n        study=study,\n        sort_order=sort_order,\n        per_fold=per_fold_str,\n    )\n</code></pre>"},{"location":"reference/evaluations/#openml.evaluations.list_evaluations_setups","title":"<code>list_evaluations_setups(function, offset=None, size=None, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, per_fold=None, sort_order=None, output_format='dataframe', parameters_in_separate_columns=False)</code>","text":"<p>List all run-evaluation pairs matching all of the given filters and their hyperparameter settings.</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>tasks</code> <code>list[int]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[int]</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>str</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dataframe'</code> <code>parameters_in_separate_columns</code> <code>bool</code> <p>Returns hyperparameters in separate columns if set to True. Valid only for a single flow</p> <code>False</code> <p>Returns:</p> Type Description <code>dict or dataframe with hyperparameter settings as a list of tuples.</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations_setups(\n    function: str,\n    offset: int | None = None,\n    size: int | None = None,\n    tasks: list | None = None,\n    setups: list | None = None,\n    flows: list | None = None,\n    runs: list | None = None,\n    uploaders: list | None = None,\n    tag: str | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: str = \"dataframe\",\n    parameters_in_separate_columns: bool = False,  # noqa: FBT001, FBT002\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters\n    and their hyperparameter settings.\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n    tasks : list[int], optional\n        the list of task IDs\n    setups: list[int], optional\n        the list of setup IDs\n    flows : list[int], optional\n        the list of flow IDs\n    runs : list[int], optional\n        the list of run IDs\n    uploaders : list[int], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n    per_fold : bool, optional\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n    output_format: str, optional (default='dataframe')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    parameters_in_separate_columns: bool, optional (default= False)\n        Returns hyperparameters in separate columns if set to True.\n        Valid only for a single flow\n\n\n    Returns\n    -------\n    dict or dataframe with hyperparameter settings as a list of tuples.\n    \"\"\"\n    if parameters_in_separate_columns and (flows is None or len(flows) != 1):\n        raise ValueError(\n            \"Can set parameters_in_separate_columns to true \" \"only for single flow_id\",\n        )\n\n    # List evaluations\n    evals = list_evaluations(\n        function=function,\n        offset=offset,\n        size=size,\n        runs=runs,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        uploaders=uploaders,\n        tag=tag,\n        per_fold=per_fold,\n        sort_order=sort_order,\n        output_format=\"dataframe\",\n    )\n    # List setups\n    # list_setups by setup id does not support large sizes (exceeds URL length limit)\n    # Hence we split the list of unique setup ids returned by list_evaluations into chunks of size N\n    _df = pd.DataFrame()\n    if len(evals) != 0:\n        N = 100  # size of section\n        length = len(evals[\"setup_id\"].unique())  # length of the array we want to split\n        # array_split - allows indices_or_sections to not equally divide the array\n        # array_split -length % N sub-arrays of size length//N + 1 and the rest of size length//N.\n        uniq = np.asarray(evals[\"setup_id\"].unique())\n        setup_chunks = np.array_split(uniq, ((length - 1) // N) + 1)\n        setup_data = pd.DataFrame()\n        for _setups in setup_chunks:\n            result = openml.setups.list_setups(setup=_setups, output_format=\"dataframe\")\n            assert isinstance(result, pd.DataFrame)\n            result = result.drop(\"flow_id\", axis=1)\n            # concat resulting setup chunks into single datframe\n            setup_data = pd.concat([setup_data, result], ignore_index=True)\n\n        parameters = []\n        # Convert parameters of setup into list of tuples of (hyperparameter, value)\n        for parameter_dict in setup_data[\"parameters\"]:\n            if parameter_dict is not None:\n                parameters.append(\n                    {param[\"full_name\"]: param[\"value\"] for param in parameter_dict.values()},\n                )\n            else:\n                parameters.append({})\n        setup_data[\"parameters\"] = parameters\n        # Merge setups with evaluations\n        _df = evals.merge(setup_data, on=\"setup_id\", how=\"left\")\n\n    if parameters_in_separate_columns:\n        _df = pd.concat(\n            [_df.drop(\"parameters\", axis=1), _df[\"parameters\"].apply(pd.Series)],\n            axis=1,\n        )\n\n    if output_format == \"dataframe\":\n        return _df\n\n    return _df.to_dict(orient=\"index\")\n</code></pre>"},{"location":"reference/evaluations/evaluation/","title":"evaluation","text":""},{"location":"reference/evaluations/evaluation/#openml.evaluations.evaluation.OpenMLEvaluation","title":"<code>OpenMLEvaluation</code>","text":"<p>Contains all meta-information about a run / evaluation combination, according to the evaluation/list function</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Refers to the run.</p> required <code>task_id</code> <code>int</code> <p>Refers to the task.</p> required <code>setup_id</code> <code>int</code> <p>Refers to the setup.</p> required <code>flow_id</code> <code>int</code> <p>Refers to the flow.</p> required <code>flow_name</code> <code>str</code> <p>Name of the referred flow.</p> required <code>data_id</code> <code>int</code> <p>Refers to the dataset.</p> required <code>data_name</code> <code>str</code> <p>The name of the dataset.</p> required <code>function</code> <code>str</code> <p>The evaluation metric of this item (e.g., accuracy).</p> required <code>upload_time</code> <code>str</code> <p>The time of evaluation.</p> required <code>uploader</code> <code>int</code> <p>Uploader ID (user ID)</p> required <code>upload_name</code> <code>str</code> <p>Name of the uploader of this evaluation</p> required <code>value</code> <code>float</code> <p>The value (score) of this evaluation.</p> required <code>values</code> <code>List[float]</code> <p>The values (scores) per repeat and fold (if requested)</p> required <code>array_data</code> <code>str</code> <p>list of information per class. (e.g., in case of precision, auroc, recall)</p> <code>None</code> Source code in <code>openml/evaluations/evaluation.py</code> <pre><code>class OpenMLEvaluation:\n    \"\"\"\n    Contains all meta-information about a run / evaluation combination,\n    according to the evaluation/list function\n\n    Parameters\n    ----------\n    run_id : int\n        Refers to the run.\n    task_id : int\n        Refers to the task.\n    setup_id : int\n        Refers to the setup.\n    flow_id : int\n        Refers to the flow.\n    flow_name : str\n        Name of the referred flow.\n    data_id : int\n        Refers to the dataset.\n    data_name : str\n        The name of the dataset.\n    function : str\n        The evaluation metric of this item (e.g., accuracy).\n    upload_time : str\n        The time of evaluation.\n    uploader: int\n        Uploader ID (user ID)\n    upload_name : str\n        Name of the uploader of this evaluation\n    value : float\n        The value (score) of this evaluation.\n    values : List[float]\n        The values (scores) per repeat and fold (if requested)\n    array_data : str\n        list of information per class.\n        (e.g., in case of precision, auroc, recall)\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        run_id: int,\n        task_id: int,\n        setup_id: int,\n        flow_id: int,\n        flow_name: str,\n        data_id: int,\n        data_name: str,\n        function: str,\n        upload_time: str,\n        uploader: int,\n        uploader_name: str,\n        value: float | None,\n        values: list[float] | None,\n        array_data: str | None = None,\n    ):\n        self.run_id = run_id\n        self.task_id = task_id\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.data_id = data_id\n        self.data_name = data_name\n        self.function = function\n        self.upload_time = upload_time\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.value = value\n        self.values = values\n        self.array_data = array_data\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Evaluation\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Upload Date\": self.upload_time,\n            \"Run ID\": self.run_id,\n            \"OpenML Run URL\": openml.runs.OpenMLRun.url_for_id(self.run_id),\n            \"Task ID\": self.task_id,\n            \"OpenML Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"OpenML Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Setup ID\": self.setup_id,\n            \"Data ID\": self.data_id,\n            \"Data Name\": self.data_name,\n            \"OpenML Data URL\": openml.datasets.OpenMLDataset.url_for_id(self.data_id),\n            \"Metric Used\": self.function,\n            \"Result\": self.value,\n        }\n\n        order = [\n            \"Uploader Date\",\n            \"Run ID\",\n            \"OpenML Run URL\",\n            \"Task ID\",\n            \"OpenML Task URL\" \"Flow ID\",\n            \"OpenML Flow URL\",\n            \"Setup ID\",\n            \"Data ID\",\n            \"Data Name\",\n            \"OpenML Data URL\",\n            \"Metric Used\",\n            \"Result\",\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/evaluations/functions/","title":"functions","text":""},{"location":"reference/evaluations/functions/#openml.evaluations.functions.__list_evaluations","title":"<code>__list_evaluations(api_call, output_format='object')</code>","text":"<p>Helper function to parse API calls which are lists of runs</p> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def __list_evaluations(\n    api_call: str,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Helper function to parse API calls which are lists of runs\"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    evals_dict = xmltodict.parse(xml_string, force_list=(\"oml:evaluation\",))\n    # Minimalistic check if the XML is useful\n    if \"oml:evaluations\" not in evals_dict:\n        raise ValueError(\n            \"Error in return XML, does not contain \" '\"oml:evaluations\": %s' % str(evals_dict),\n        )\n\n    assert isinstance(evals_dict[\"oml:evaluations\"][\"oml:evaluation\"], list), type(\n        evals_dict[\"oml:evaluations\"],\n    )\n\n    evals: dict[int, dict | OpenMLEvaluation] = {}\n    uploader_ids = list(\n        {eval_[\"oml:uploader\"] for eval_ in evals_dict[\"oml:evaluations\"][\"oml:evaluation\"]},\n    )\n    api_users = \"user/list/user_id/\" + \",\".join(uploader_ids)\n    xml_string_user = openml._api_calls._perform_api_call(api_users, \"get\")\n    users = xmltodict.parse(xml_string_user, force_list=(\"oml:user\",))\n    user_dict = {user[\"oml:id\"]: user[\"oml:username\"] for user in users[\"oml:users\"][\"oml:user\"]}\n    for eval_ in evals_dict[\"oml:evaluations\"][\"oml:evaluation\"]:\n        run_id = int(eval_[\"oml:run_id\"])\n\n        value = None\n        if \"oml:value\" in eval_:\n            value = float(eval_[\"oml:value\"])\n\n        values = None\n        if \"oml:values\" in eval_:\n            values = json.loads(eval_[\"oml:values\"])\n\n        array_data = eval_.get(\"oml:array_data\")\n\n        if output_format == \"object\":\n            evals[run_id] = OpenMLEvaluation(\n                run_id=run_id,\n                task_id=int(eval_[\"oml:task_id\"]),\n                setup_id=int(eval_[\"oml:setup_id\"]),\n                flow_id=int(eval_[\"oml:flow_id\"]),\n                flow_name=eval_[\"oml:flow_name\"],\n                data_id=int(eval_[\"oml:data_id\"]),\n                data_name=eval_[\"oml:data_name\"],\n                function=eval_[\"oml:function\"],\n                upload_time=eval_[\"oml:upload_time\"],\n                uploader=int(eval_[\"oml:uploader\"]),\n                uploader_name=user_dict[eval_[\"oml:uploader\"]],\n                value=value,\n                values=values,\n                array_data=array_data,\n            )\n        else:\n            # for output_format in ['dict', 'dataframe']\n            evals[run_id] = {\n                \"run_id\": int(eval_[\"oml:run_id\"]),\n                \"task_id\": int(eval_[\"oml:task_id\"]),\n                \"setup_id\": int(eval_[\"oml:setup_id\"]),\n                \"flow_id\": int(eval_[\"oml:flow_id\"]),\n                \"flow_name\": eval_[\"oml:flow_name\"],\n                \"data_id\": int(eval_[\"oml:data_id\"]),\n                \"data_name\": eval_[\"oml:data_name\"],\n                \"function\": eval_[\"oml:function\"],\n                \"upload_time\": eval_[\"oml:upload_time\"],\n                \"uploader\": int(eval_[\"oml:uploader\"]),\n                \"uploader_name\": user_dict[eval_[\"oml:uploader\"]],\n                \"value\": value,\n                \"values\": values,\n                \"array_data\": array_data,\n            }\n\n    if output_format == \"dataframe\":\n        rows = list(evals.values())\n        return pd.DataFrame.from_records(rows, columns=rows[0].keys())  # type: ignore\n\n    return evals\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_estimation_procedures","title":"<code>list_estimation_procedures()</code>","text":"<p>Return list of evaluation procedures available.</p> <p>The function performs an API call to retrieve the entire list of evaluation procedures' names that are available.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_estimation_procedures() -&gt; list[str]:\n    \"\"\"Return list of evaluation procedures available.\n\n    The function performs an API call to retrieve the entire list of\n    evaluation procedures' names that are available.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    api_call = \"estimationprocedure/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    api_results = xmltodict.parse(xml_string)\n\n    # Minimalistic check if the XML is useful\n    if \"oml:estimationprocedures\" not in api_results:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:estimationprocedures\"')\n    if \"oml:estimationprocedure\" not in api_results[\"oml:estimationprocedures\"]:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:estimationprocedure\"')\n\n    if not isinstance(api_results[\"oml:estimationprocedures\"][\"oml:estimationprocedure\"], list):\n        raise TypeError(\n            \"Error in return XML, does not contain \" '\"oml:estimationprocedure\" as a list',\n        )\n\n    return [\n        prod[\"oml:name\"]\n        for prod in api_results[\"oml:estimationprocedures\"][\"oml:estimationprocedure\"]\n    ]\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_evaluation_measures","title":"<code>list_evaluation_measures()</code>","text":"<p>Return list of evaluation measures available.</p> <p>The function performs an API call to retrieve the entire list of evaluation measures that are available.</p> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluation_measures() -&gt; list[str]:\n    \"\"\"Return list of evaluation measures available.\n\n    The function performs an API call to retrieve the entire list of\n    evaluation measures that are available.\n\n    Returns\n    -------\n    list\n\n    \"\"\"\n    api_call = \"evaluationmeasure/list\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    qualities = xmltodict.parse(xml_string, force_list=(\"oml:measures\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:evaluation_measures\" not in qualities:\n        raise ValueError(\"Error in return XML, does not contain \" '\"oml:evaluation_measures\"')\n    if not isinstance(qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"], list):\n        raise TypeError(\"Error in return XML, does not contain \" '\"oml:measure\" as a list')\n    return qualities[\"oml:evaluation_measures\"][\"oml:measures\"][0][\"oml:measure\"]\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_evaluations","title":"<code>list_evaluations(function, offset=None, size=10000, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, study=None, per_fold=None, sort_order=None, output_format='object')</code>","text":"<pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dict', 'object'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_evaluations(function: str, offset: int | None = ..., size: int | None = ..., tasks: list[str | int] | None = ..., setups: list[str | int] | None = ..., flows: list[str | int] | None = ..., runs: list[str | int] | None = ..., uploaders: list[str | int] | None = ..., tag: str | None = ..., study: int | None = ..., per_fold: bool | None = ..., sort_order: str | None = ..., output_format: Literal['dataframe'] = ...) -&gt; pd.DataFrame\n</code></pre> <p>List all run-evaluation pairs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of runs to show. If set to <code>None</code>, it returns all the results.</p> <code>10000</code> <code>tasks</code> <code>list[int, str]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list[str | int] | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int, str]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[str | int] | None</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int, str]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'object' the output is a dict of OpenMLEvaluation objects - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations(\n    function: str,\n    offset: int | None = None,\n    size: int | None = 10000,\n    tasks: list[str | int] | None = None,\n    setups: list[str | int] | None = None,\n    flows: list[str | int] | None = None,\n    runs: list[str | int] | None = None,\n    uploaders: list[str | int] | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, default 10000\n        The maximum number of runs to show.\n        If set to ``None``, it returns all the results.\n\n    tasks : list[int,str], optional\n        the list of task IDs\n    setups: list[int,str], optional\n        the list of setup IDs\n    flows : list[int,str], optional\n        the list of flow IDs\n    runs :list[int,str], optional\n        the list of run IDs\n    uploaders : list[int,str], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n\n    study : int, optional\n\n    per_fold : bool, optional\n\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'object' the output is a dict of OpenMLEvaluation objects\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. Only 'object', 'dataframe', or 'dict' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    per_fold_str = None\n    if per_fold is not None:\n        per_fold_str = str(per_fold).lower()\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_evaluations,\n        function=function,\n        offset=offset,\n        size=size,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        runs=runs,\n        uploaders=uploaders,\n        tag=tag,\n        study=study,\n        sort_order=sort_order,\n        per_fold=per_fold_str,\n    )\n</code></pre>"},{"location":"reference/evaluations/functions/#openml.evaluations.functions.list_evaluations_setups","title":"<code>list_evaluations_setups(function, offset=None, size=None, tasks=None, setups=None, flows=None, runs=None, uploaders=None, tag=None, per_fold=None, sort_order=None, output_format='dataframe', parameters_in_separate_columns=False)</code>","text":"<p>List all run-evaluation pairs matching all of the given filters and their hyperparameter settings.</p> <p>Parameters:</p> Name Type Description Default <code>function</code> <code>str</code> <p>the evaluation function. e.g., predictive_accuracy</p> required <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>tasks</code> <code>list[int]</code> <p>the list of task IDs</p> <code>None</code> <code>setups</code> <code>list | None</code> <p>the list of setup IDs</p> <code>None</code> <code>flows</code> <code>list[int]</code> <p>the list of flow IDs</p> <code>None</code> <code>runs</code> <code>list[int]</code> <p>the list of run IDs</p> <code>None</code> <code>uploaders</code> <code>list[int]</code> <p>the list of uploader IDs</p> <code>None</code> <code>tag</code> <code>str</code> <p>filter evaluation based on given tag</p> <code>None</code> <code>per_fold</code> <code>bool</code> <code>None</code> <code>sort_order</code> <code>str</code> <p>order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")</p> <code>None</code> <code>output_format</code> <code>str</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dataframe'</code> <code>parameters_in_separate_columns</code> <code>bool</code> <p>Returns hyperparameters in separate columns if set to True. Valid only for a single flow</p> <code>False</code> <p>Returns:</p> Type Description <code>dict or dataframe with hyperparameter settings as a list of tuples.</code> Source code in <code>openml/evaluations/functions.py</code> <pre><code>def list_evaluations_setups(\n    function: str,\n    offset: int | None = None,\n    size: int | None = None,\n    tasks: list | None = None,\n    setups: list | None = None,\n    flows: list | None = None,\n    runs: list | None = None,\n    uploaders: list | None = None,\n    tag: str | None = None,\n    per_fold: bool | None = None,\n    sort_order: str | None = None,\n    output_format: str = \"dataframe\",\n    parameters_in_separate_columns: bool = False,  # noqa: FBT001, FBT002\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all run-evaluation pairs matching all of the given filters\n    and their hyperparameter settings.\n\n    Parameters\n    ----------\n    function : str\n        the evaluation function. e.g., predictive_accuracy\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n    tasks : list[int], optional\n        the list of task IDs\n    setups: list[int], optional\n        the list of setup IDs\n    flows : list[int], optional\n        the list of flow IDs\n    runs : list[int], optional\n        the list of run IDs\n    uploaders : list[int], optional\n        the list of uploader IDs\n    tag : str, optional\n        filter evaluation based on given tag\n    per_fold : bool, optional\n    sort_order : str, optional\n       order of sorting evaluations, ascending (\"asc\") or descending (\"desc\")\n    output_format: str, optional (default='dataframe')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    parameters_in_separate_columns: bool, optional (default= False)\n        Returns hyperparameters in separate columns if set to True.\n        Valid only for a single flow\n\n\n    Returns\n    -------\n    dict or dataframe with hyperparameter settings as a list of tuples.\n    \"\"\"\n    if parameters_in_separate_columns and (flows is None or len(flows) != 1):\n        raise ValueError(\n            \"Can set parameters_in_separate_columns to true \" \"only for single flow_id\",\n        )\n\n    # List evaluations\n    evals = list_evaluations(\n        function=function,\n        offset=offset,\n        size=size,\n        runs=runs,\n        tasks=tasks,\n        setups=setups,\n        flows=flows,\n        uploaders=uploaders,\n        tag=tag,\n        per_fold=per_fold,\n        sort_order=sort_order,\n        output_format=\"dataframe\",\n    )\n    # List setups\n    # list_setups by setup id does not support large sizes (exceeds URL length limit)\n    # Hence we split the list of unique setup ids returned by list_evaluations into chunks of size N\n    _df = pd.DataFrame()\n    if len(evals) != 0:\n        N = 100  # size of section\n        length = len(evals[\"setup_id\"].unique())  # length of the array we want to split\n        # array_split - allows indices_or_sections to not equally divide the array\n        # array_split -length % N sub-arrays of size length//N + 1 and the rest of size length//N.\n        uniq = np.asarray(evals[\"setup_id\"].unique())\n        setup_chunks = np.array_split(uniq, ((length - 1) // N) + 1)\n        setup_data = pd.DataFrame()\n        for _setups in setup_chunks:\n            result = openml.setups.list_setups(setup=_setups, output_format=\"dataframe\")\n            assert isinstance(result, pd.DataFrame)\n            result = result.drop(\"flow_id\", axis=1)\n            # concat resulting setup chunks into single datframe\n            setup_data = pd.concat([setup_data, result], ignore_index=True)\n\n        parameters = []\n        # Convert parameters of setup into list of tuples of (hyperparameter, value)\n        for parameter_dict in setup_data[\"parameters\"]:\n            if parameter_dict is not None:\n                parameters.append(\n                    {param[\"full_name\"]: param[\"value\"] for param in parameter_dict.values()},\n                )\n            else:\n                parameters.append({})\n        setup_data[\"parameters\"] = parameters\n        # Merge setups with evaluations\n        _df = evals.merge(setup_data, on=\"setup_id\", how=\"left\")\n\n    if parameters_in_separate_columns:\n        _df = pd.concat(\n            [_df.drop(\"parameters\", axis=1), _df[\"parameters\"].apply(pd.Series)],\n            axis=1,\n        )\n\n    if output_format == \"dataframe\":\n        return _df\n\n    return _df.to_dict(orient=\"index\")\n</code></pre>"},{"location":"reference/extensions/","title":"extensions","text":""},{"location":"reference/extensions/#openml.extensions.Extension","title":"<code>Extension</code>","text":"<p>               Bases: <code>ABC</code></p> <p>Defines the interface to connect machine learning libraries to OpenML-Python.</p> <p>See <code>openml.extension.sklearn.extension</code> for an implementation to bootstrap from.</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>class Extension(ABC):\n    \"\"\"Defines the interface to connect machine learning libraries to OpenML-Python.\n\n    See ``openml.extension.sklearn.extension`` for an implementation to bootstrap from.\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    @abstractmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given flow can be handled by this extension.\n\n        This is typically done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @classmethod\n    @abstractmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model flow can be handled by this extension.\n\n        This is typically done by checking the type of the model, or the package it belongs to.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for flow serialization and de-serialization\n\n    @abstractmethod\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Instantiate a model from the flow representation.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n    @abstractmethod\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n\n    @abstractmethod\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Returns\n        -------\n        List\n        \"\"\"\n\n    @abstractmethod\n    def create_setup_string(self, model: Any) -&gt; str:\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for performing runs with extension modules\n\n    @abstractmethod\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is an estimator for the given extension.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @abstractmethod\n    def seed_model(self, model: Any, seed: int | None) -&gt; Any:\n        \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Parameters\n        ----------\n        model : Any\n            The model to be seeded\n        seed : int\n\n        Returns\n        -------\n        model\n        \"\"\"\n\n    @abstractmethod\n    def _run_model_on_fold(  # noqa: PLR0913\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | None = None,\n    ) -&gt; tuple[np.ndarray, np.ndarray | None, OrderedDict[str, float], OpenMLRunTrace | None]:\n        \"\"\"Run a model on a repeat, fold, subsample triplet of the task.\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        :func:`openml.runs.run_flow_on_task`.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        predictions : np.ndarray\n            Model predictions.\n        probabilities :  Optional, np.ndarray\n            Predicted probabilities (only applicable for supervised classification tasks).\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : Optional, OpenMLRunTrace\n            Hyperparameter optimization trace (only applicable for supervised tasks with\n            hyperparameter optimization).\n        \"\"\"\n\n    @abstractmethod\n    def obtain_parameter_values(\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n\n    @abstractmethod\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for hyperparameter optimization\n\n    @abstractmethod\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n        model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a given flow can be handled by this extension.</p> <p>This is typically done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given flow can be handled by this extension.\n\n    This is typically done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a model flow can be handled by this extension.</p> <p>This is typically done by checking the type of the model, or the package it belongs to.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model flow can be handled by this extension.\n\n    This is typically done by checking the type of the model, or the package it belongs to.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>  <code>abstractmethod</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.create_setup_string","title":"<code>create_setup_string(model)</code>  <code>abstractmethod</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef create_setup_string(self, model: Any) -&gt; str:\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a model from the flow representation.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT002, FBT001\n) -&gt; Any:\n    \"\"\"Instantiate a model from the flow representation.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.get_version_information","title":"<code>get_version_information()</code>  <code>abstractmethod</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Returns\n    -------\n    List\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a base model which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n    model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.is_estimator","title":"<code>is_estimator(model)</code>  <code>abstractmethod</code>","text":"<p>Check whether the given model is an estimator for the given extension.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is an estimator for the given extension.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.model_to_flow","title":"<code>model_to_flow(model)</code>  <code>abstractmethod</code>","text":"<p>Transform a model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>  <code>abstractmethod</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef obtain_parameter_values(\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.Extension.seed_model","title":"<code>seed_model(model, seed)</code>  <code>abstractmethod</code>","text":"<p>Set the seed of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef seed_model(self, model: Any, seed: int | None) -&gt; Any:\n    \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Parameters\n    ----------\n    model : Any\n        The model to be seeded\n    seed : int\n\n    Returns\n    -------\n    model\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.get_extension_by_flow","title":"<code>get_extension_by_flow(flow, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented flow. Raises an exception if two extensions can handle a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented flow.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_flow(\n    flow: OpenMLFlow,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented flow.\n    Raises an exception if two extensions can handle a flow.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented flow.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_flow(flow):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle flow: {flow}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle flow: {flow}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.get_extension_by_model","title":"<code>get_extension_by_model(model, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented model. Raises an exception if two extensions can handle a model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented model.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_model(\n    model: Any,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented model.\n    Raises an exception if two extensions can handle a model.\n\n    Parameters\n    ----------\n    model : Any\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented model.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_model(model):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle model: {model}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle model: {model}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/#openml.extensions.register_extension","title":"<code>register_extension(extension)</code>","text":"<p>Register an extension.</p> <p>Registered extensions are considered by <code>get_extension_by_flow</code> and <code>get_extension_by_model</code>, which are used by <code>openml.flow</code> and <code>openml.runs</code>.</p> <p>Parameters:</p> Name Type Description Default <code>extension</code> <code>Type[Extension]</code> required <p>Returns:</p> Type Description <code>None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def register_extension(extension: type[Extension]) -&gt; None:\n    \"\"\"Register an extension.\n\n    Registered extensions are considered by ``get_extension_by_flow`` and\n    ``get_extension_by_model``, which are used by ``openml.flow`` and ``openml.runs``.\n\n    Parameters\n    ----------\n    extension : Type[Extension]\n\n    Returns\n    -------\n    None\n    \"\"\"\n    openml.extensions.extensions.append(extension)\n</code></pre>"},{"location":"reference/extensions/extension_interface/","title":"extension_interface","text":""},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension","title":"<code>Extension</code>","text":"<p>               Bases: <code>ABC</code></p> <p>Defines the interface to connect machine learning libraries to OpenML-Python.</p> <p>See <code>openml.extension.sklearn.extension</code> for an implementation to bootstrap from.</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>class Extension(ABC):\n    \"\"\"Defines the interface to connect machine learning libraries to OpenML-Python.\n\n    See ``openml.extension.sklearn.extension`` for an implementation to bootstrap from.\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    @abstractmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given flow can be handled by this extension.\n\n        This is typically done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @classmethod\n    @abstractmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model flow can be handled by this extension.\n\n        This is typically done by checking the type of the model, or the package it belongs to.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for flow serialization and de-serialization\n\n    @abstractmethod\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Instantiate a model from the flow representation.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n    @abstractmethod\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n\n    @abstractmethod\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Returns\n        -------\n        List\n        \"\"\"\n\n    @abstractmethod\n    def create_setup_string(self, model: Any) -&gt; str:\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for performing runs with extension modules\n\n    @abstractmethod\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is an estimator for the given extension.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    @abstractmethod\n    def seed_model(self, model: Any, seed: int | None) -&gt; Any:\n        \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Parameters\n        ----------\n        model : Any\n            The model to be seeded\n        seed : int\n\n        Returns\n        -------\n        model\n        \"\"\"\n\n    @abstractmethod\n    def _run_model_on_fold(  # noqa: PLR0913\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | None = None,\n    ) -&gt; tuple[np.ndarray, np.ndarray | None, OrderedDict[str, float], OpenMLRunTrace | None]:\n        \"\"\"Run a model on a repeat, fold, subsample triplet of the task.\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        :func:`openml.runs.run_flow_on_task`.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        predictions : np.ndarray\n            Model predictions.\n        probabilities :  Optional, np.ndarray\n            Predicted probabilities (only applicable for supervised classification tasks).\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : Optional, OpenMLRunTrace\n            Hyperparameter optimization trace (only applicable for supervised tasks with\n            hyperparameter optimization).\n        \"\"\"\n\n    @abstractmethod\n    def obtain_parameter_values(\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n\n    @abstractmethod\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n\n    ################################################################################################\n    # Abstract methods for hyperparameter optimization\n\n    @abstractmethod\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n        model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a given flow can be handled by this extension.</p> <p>This is typically done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given flow can be handled by this extension.\n\n    This is typically done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>abstractmethod</code> <code>classmethod</code>","text":"<p>Check whether a model flow can be handled by this extension.</p> <p>This is typically done by checking the type of the model, or the package it belongs to.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@classmethod\n@abstractmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model flow can be handled by this extension.\n\n    This is typically done by checking the type of the model, or the package it belongs to.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>  <code>abstractmethod</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.create_setup_string","title":"<code>create_setup_string(model)</code>  <code>abstractmethod</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef create_setup_string(self, model: Any) -&gt; str:\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a model from the flow representation.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT002, FBT001\n) -&gt; Any:\n    \"\"\"Instantiate a model from the flow representation.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.get_version_information","title":"<code>get_version_information()</code>  <code>abstractmethod</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Returns\n    -------\n    List\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>  <code>abstractmethod</code>","text":"<p>Instantiate a base model which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a base model which can be searched over by the hyperparameter optimization\n    model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.is_estimator","title":"<code>is_estimator(model)</code>  <code>abstractmethod</code>","text":"<p>Check whether the given model is an estimator for the given extension.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is an estimator for the given extension.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.model_to_flow","title":"<code>model_to_flow(model)</code>  <code>abstractmethod</code>","text":"<p>Transform a model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>  <code>abstractmethod</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef obtain_parameter_values(\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension.seed_model","title":"<code>seed_model(model, seed)</code>  <code>abstractmethod</code>","text":"<p>Set the seed of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/extensions/extension_interface.py</code> <pre><code>@abstractmethod\ndef seed_model(self, model: Any, seed: int | None) -&gt; Any:\n    \"\"\"Set the seed of all the unseeded components of a model and return the seeded model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Parameters\n    ----------\n    model : Any\n        The model to be seeded\n    seed : int\n\n    Returns\n    -------\n    model\n    \"\"\"\n</code></pre>"},{"location":"reference/extensions/functions/","title":"functions","text":""},{"location":"reference/extensions/functions/#openml.extensions.functions.get_extension_by_flow","title":"<code>get_extension_by_flow(flow, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented flow. Raises an exception if two extensions can handle a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented flow.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_flow(\n    flow: OpenMLFlow,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented flow.\n    Raises an exception if two extensions can handle a flow.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented flow.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_flow(flow):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle flow: {flow}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle flow: {flow}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/functions/#openml.extensions.functions.get_extension_by_model","title":"<code>get_extension_by_model(model, raise_if_no_extension=False)</code>","text":"<p>Get an extension which can handle the given flow.</p> <p>Iterates all registered extensions and checks whether they can handle the presented model. Raises an exception if two extensions can handle a model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <code>raise_if_no_extension</code> <code>bool (optional</code> <p>Raise an exception if no registered extension can handle the presented model.</p> <code>False)</code> <p>Returns:</p> Type Description <code>Extension or None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def get_extension_by_model(\n    model: Any,\n    raise_if_no_extension: bool = False,  # noqa: FBT001, FBT002\n) -&gt; Extension | None:\n    \"\"\"Get an extension which can handle the given flow.\n\n    Iterates all registered extensions and checks whether they can handle the presented model.\n    Raises an exception if two extensions can handle a model.\n\n    Parameters\n    ----------\n    model : Any\n\n    raise_if_no_extension : bool (optional, default=False)\n        Raise an exception if no registered extension can handle the presented model.\n\n    Returns\n    -------\n    Extension or None\n    \"\"\"\n    candidates = []\n    for extension_class in openml.extensions.extensions:\n        if extension_class.can_handle_model(model):\n            candidates.append(extension_class())\n    if len(candidates) == 0:\n        if raise_if_no_extension:\n            raise ValueError(f\"No extension registered which can handle model: {model}\")\n\n        return None\n\n    if len(candidates) == 1:\n        return candidates[0]\n\n    raise ValueError(\n        f\"Multiple extensions registered which can handle model: {model}, but only one \"\n        f\"is allowed ({candidates}).\",\n    )\n</code></pre>"},{"location":"reference/extensions/functions/#openml.extensions.functions.register_extension","title":"<code>register_extension(extension)</code>","text":"<p>Register an extension.</p> <p>Registered extensions are considered by <code>get_extension_by_flow</code> and <code>get_extension_by_model</code>, which are used by <code>openml.flow</code> and <code>openml.runs</code>.</p> <p>Parameters:</p> Name Type Description Default <code>extension</code> <code>Type[Extension]</code> required <p>Returns:</p> Type Description <code>None</code> Source code in <code>openml/extensions/functions.py</code> <pre><code>def register_extension(extension: type[Extension]) -&gt; None:\n    \"\"\"Register an extension.\n\n    Registered extensions are considered by ``get_extension_by_flow`` and\n    ``get_extension_by_model``, which are used by ``openml.flow`` and ``openml.runs``.\n\n    Parameters\n    ----------\n    extension : Type[Extension]\n\n    Returns\n    -------\n    None\n    \"\"\"\n    openml.extensions.extensions.append(extension)\n</code></pre>"},{"location":"reference/extensions/sklearn/","title":"sklearn","text":""},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension","title":"<code>SklearnExtension</code>","text":"<p>               Bases: <code>Extension</code></p> <p>Connect scikit-learn to OpenML-Python. The estimators which use this extension must be scikit-learn compatible, i.e needs to be a subclass of sklearn.base.BaseEstimator\".</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>class SklearnExtension(Extension):\n    \"\"\"Connect scikit-learn to OpenML-Python.\n    The estimators which use this extension must be scikit-learn compatible,\n    i.e needs to be a subclass of sklearn.base.BaseEstimator\".\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given describes a scikit-learn estimator.\n\n        This is done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return cls._is_sklearn_flow(flow)\n\n    @classmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.base.BaseEstimator)\n\n    @classmethod\n    def trim_flow_name(  # noqa: C901\n        cls,\n        long_name: str,\n        extra_trim_length: int = 100,\n        _outer: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; str:\n        \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n        Flows are assumed to have the following naming structure:\n        ``(model_selection)? (pipeline)? (steps)+``\n        and will be shortened to:\n        ``sklearn.(selection.)?(pipeline.)?(steps)+``\n        e.g. (white spaces and newlines added for readability)\n\n        .. code ::\n\n            sklearn.pipeline.Pipeline(\n                columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                    numeric=sklearn.pipeline.Pipeline(\n                        imputer=sklearn.preprocessing.imputation.Imputer,\n                        standardscaler=sklearn.preprocessing.data.StandardScaler),\n                    nominal=sklearn.pipeline.Pipeline(\n                        simpleimputer=sklearn.impute.SimpleImputer,\n                        onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n                variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n                svc=sklearn.svm.classes.SVC)\n\n        -&gt;\n        ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n        Parameters\n        ----------\n        long_name : str\n            The full flow name generated by the scikit-learn extension.\n        extra_trim_length: int (default=100)\n            If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n            of the short name is performed. This reduces the produced short name length.\n            There is no guarantee the end result will not exceed `extra_trim_length`.\n        _outer : bool (default=True)\n            For internal use only. Specifies if the function is called recursively.\n\n        Returns\n        -------\n        str\n\n        \"\"\"\n\n        def remove_all_in_parentheses(string: str) -&gt; str:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            while removals &gt; 0:\n                string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            return string\n\n        # Generally, we want to trim all hyperparameters, the exception to that is for model\n        # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n        # So we first trim name of the `estimator` specified in mode selection. For reference, in\n        # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n        # keep it in the final trimmed flow name:\n        # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n        # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n        # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n        # sklearn.tree.tree.DecisionTreeClassifier))\n        if \"sklearn.model_selection\" in long_name:\n            start_index = long_name.index(\"sklearn.model_selection\")\n            estimator_start = (\n                start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n            )\n\n            model_select_boilerplate = long_name[start_index:estimator_start]\n            # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n            model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n            # Now we want to also find and parse the `estimator`, for this we find the closing\n            # parenthesis to the model selection technique:\n            closing_parenthesis_expected = 1\n            for char in long_name[estimator_start:]:\n                if char == \"(\":\n                    closing_parenthesis_expected += 1\n                if char == \")\":\n                    closing_parenthesis_expected -= 1\n                if closing_parenthesis_expected == 0:\n                    break\n\n            _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n            model_select_pipeline = long_name[estimator_start:_end]\n\n            trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n            _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n            model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n            name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n        else:\n            name = long_name\n\n        module_name = long_name.split(\".\")[0]\n        short_name = module_name + \".{}\"\n\n        if name.startswith(\"sklearn.pipeline\"):\n            full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n            pipeline_class = full_pipeline_class.split(\".\")[-1]\n            # We don't want nested pipelines in the short name, so we trim all complicated\n            # subcomponents, i.e. those with parentheses:\n            pipeline = remove_all_in_parentheses(pipeline)\n\n            # then the pipeline steps are formatted e.g.:\n            # step1name=sklearn.submodule.ClassName,step2name...\n            components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n            pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n            if len(short_name.format(pipeline)) &gt; extra_trim_length:\n                pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n        else:\n            # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n            pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n        if not _outer:\n            # Anything from parenthesis in inner calls should not be culled, so we use brackets\n            pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n        else:\n            # Square brackets may be introduced with nested model_selection\n            pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n        return short_name.format(pipeline)\n\n    @classmethod\n    def _min_dependency_str(cls, sklearn_version: str) -&gt; str:\n        \"\"\"Returns a string containing the minimum dependencies for the sklearn version passed.\n\n        Parameters\n        ----------\n        sklearn_version : str\n            A version string of the xx.xx.xx\n\n        Returns\n        -------\n        str\n        \"\"\"\n        openml_major_version = int(LooseVersion(openml.__version__).version[1])\n        # This explicit check is necessary to support existing entities on the OpenML servers\n        # that used the fixed dependency string (in the else block)\n        if openml_major_version &gt; 11:\n            # OpenML v0.11 onwards supports sklearn&gt;=0.24\n            # assumption: 0.24 onwards sklearn should contain a _min_dependencies.py file with\n            # variables declared for extracting minimum dependency for that version\n            if LooseVersion(sklearn_version) &gt;= \"0.24\":\n                from sklearn import _min_dependencies as _mindep\n\n                dependency_list = {\n                    \"numpy\": f\"{_mindep.NUMPY_MIN_VERSION}\",\n                    \"scipy\": f\"{_mindep.SCIPY_MIN_VERSION}\",\n                    \"joblib\": f\"{_mindep.JOBLIB_MIN_VERSION}\",\n                    \"threadpoolctl\": f\"{_mindep.THREADPOOLCTL_MIN_VERSION}\",\n                }\n            elif LooseVersion(sklearn_version) &gt;= \"0.23\":\n                dependency_list = {\n                    \"numpy\": \"1.13.3\",\n                    \"scipy\": \"0.19.1\",\n                    \"joblib\": \"0.11\",\n                    \"threadpoolctl\": \"2.0.0\",\n                }\n                if LooseVersion(sklearn_version).version[2] == 0:\n                    dependency_list.pop(\"threadpoolctl\")\n            elif LooseVersion(sklearn_version) &gt;= \"0.21\":\n                dependency_list = {\"numpy\": \"1.11.0\", \"scipy\": \"0.17.0\", \"joblib\": \"0.11\"}\n            elif LooseVersion(sklearn_version) &gt;= \"0.19\":\n                dependency_list = {\"numpy\": \"1.8.2\", \"scipy\": \"0.13.3\"}\n            else:\n                dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n        else:\n            # this is INCORRECT for sklearn versions &gt;= 0.19 and &lt; 0.24\n            # given that OpenML has existing flows uploaded with such dependency information,\n            # we change no behaviour for older sklearn version, however from 0.24 onwards\n            # the dependency list will be accurately updated for any flow uploaded to OpenML\n            dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n\n        sklearn_dep = f\"sklearn=={sklearn_version}\"\n        dep_str = \"\\n\".join([f\"{k}&gt;={v}\" for k, v in dependency_list.items()])\n        return \"\\n\".join([sklearn_dep, dep_str])\n\n    ################################################################################################\n    # Methods for flow serialization and de-serialization\n\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; Any:\n        \"\"\"Initializes a sklearn model based on a flow.\n\n        Parameters\n        ----------\n        flow : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        return self._deserialize_sklearn(\n            flow,\n            initialize_with_defaults=initialize_with_defaults,\n            strict_version=strict_version,\n        )\n\n    def _deserialize_sklearn(  # noqa: PLR0915, C901, PLR0913, PLR0912\n        self,\n        o: Any,\n        components: dict | None = None,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        recursion_depth: int = 0,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Recursive function to deserialize a scikit-learn flow.\n\n        This function inspects an object to deserialize and decides how to do so. This function\n        delegates all work to the respective functions to deserialize special data structures etc.\n        This function works on everything that has been serialized to OpenML: OpenMLFlow,\n        components (which are flows themselves), functions, hyperparameter distributions (for\n        random search) and the actual hyperparameter values themselves.\n\n        Parameters\n        ----------\n        o : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        components : Optional[dict]\n            Components of the current flow being de-serialized. These will not be used when\n            de-serializing the actual flow, but when de-serializing a component reference.\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        recursion_depth : int\n            The depth at which this flow is called, mostly for debugging\n            purposes\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        logger.info(\n            \"-{} flow_to_sklearn START o={}, components={}, init_defaults={}\".format(\n                \"-\" * recursion_depth, o, components, initialize_with_defaults\n            ),\n        )\n        depth_pp = recursion_depth + 1  # shortcut var, depth plus plus\n\n        # First, we need to check whether the presented object is a json string.\n        # JSON strings are used to encoder parameter values. By passing around\n        # json strings for parameters, we make sure that we can flow_to_sklearn\n        # the parameter values to the correct type.\n\n        if isinstance(o, str):\n            with contextlib.suppress(JSONDecodeError):\n                o = json.loads(o)\n\n        if isinstance(o, dict):\n            # Check if the dict encodes a 'special' object, which could not\n            # easily converted into a string, but rather the information to\n            # re-create the object were stored in a dictionary.\n            if \"oml-python:serialized_object\" in o:\n                serialized_type = o[\"oml-python:serialized_object\"]\n                value = o[\"value\"]\n                if serialized_type == \"type\":\n                    rval = self._deserialize_type(value)\n                elif serialized_type == \"rv_frozen\":\n                    rval = self._deserialize_rv_frozen(value)\n                elif serialized_type == \"function\":\n                    rval = self._deserialize_function(value)\n                elif serialized_type in (COMPOSITION_STEP_CONSTANT, COMPONENT_REFERENCE):\n                    if serialized_type == COMPOSITION_STEP_CONSTANT:\n                        pass\n                    elif serialized_type == COMPONENT_REFERENCE:\n                        value = self._deserialize_sklearn(\n                            value,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        )\n                    else:\n                        raise NotImplementedError(serialized_type)\n                    assert components is not None  # Necessary for mypy\n                    step_name = value[\"step_name\"]\n                    key = value[\"key\"]\n                    component = self._deserialize_sklearn(\n                        components[key],\n                        initialize_with_defaults=initialize_with_defaults,\n                        recursion_depth=depth_pp,\n                        strict_version=strict_version,\n                    )\n                    # The component is now added to where it should be used\n                    # later. It should not be passed to the constructor of the\n                    # main flow object.\n                    del components[key]\n                    if step_name is None:\n                        rval = component\n                    elif \"argument_1\" not in value:\n                        rval = (step_name, component)\n                    else:\n                        rval = (step_name, component, value[\"argument_1\"])\n                elif serialized_type == \"cv_object\":\n                    rval = self._deserialize_cross_validator(\n                        value,\n                        recursion_depth=recursion_depth,\n                        strict_version=strict_version,\n                    )\n                else:\n                    raise ValueError(\"Cannot flow_to_sklearn %s\" % serialized_type)\n\n            else:\n                rval = OrderedDict(\n                    (\n                        self._deserialize_sklearn(\n                            o=key,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                        self._deserialize_sklearn(\n                            o=value,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                    )\n                    for key, value in sorted(o.items())\n                )\n        elif isinstance(o, (list, tuple)):\n            rval = [\n                self._deserialize_sklearn(\n                    o=element,\n                    components=components,\n                    initialize_with_defaults=initialize_with_defaults,\n                    recursion_depth=depth_pp,\n                    strict_version=strict_version,\n                )\n                for element in o\n            ]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, (bool, int, float, str)) or o is None:\n            rval = o\n        elif isinstance(o, OpenMLFlow):\n            if not self._is_sklearn_flow(o):\n                raise ValueError(\"Only sklearn flows can be reinstantiated\")\n            rval = self._deserialize_model(\n                flow=o,\n                keep_defaults=initialize_with_defaults,\n                recursion_depth=recursion_depth,\n                strict_version=strict_version,\n            )\n        else:\n            raise TypeError(o)\n        logger.info(\n            \"-{} flow_to_sklearn END   o={}, rval={}\".format(\"-\" * recursion_depth, o, rval)\n        )\n        return rval\n\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n        # Necessary to make pypy not complain about all the different possible return types\n        return self._serialize_sklearn(model)\n\n    def _serialize_sklearn(self, o: Any, parent_model: Any | None = None) -&gt; Any:  # noqa: PLR0912, C901\n        rval = None  # type: Any\n\n        # TODO: assert that only on first recursion lvl `parent_model` can be None\n        if self.is_estimator(o):\n            # is the main model or a submodel\n            rval = self._serialize_model(o)\n        elif (\n            isinstance(o, (list, tuple))\n            and len(o) == 2\n            and o[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            and isinstance(parent_model, sklearn.pipeline._BaseComposition)\n        ):\n            rval = o\n        elif isinstance(o, (list, tuple)):\n            # TODO: explain what type of parameter is here\n            rval = [self._serialize_sklearn(element, parent_model) for element in o]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, SIMPLE_TYPES) or o is None:\n            if isinstance(o, tuple(SIMPLE_NUMPY_TYPES)):\n                o = o.item()  # type: ignore\n            # base parameter values\n            rval = o\n        elif isinstance(o, dict):\n            # TODO: explain what type of parameter is here\n            if not isinstance(o, OrderedDict):\n                o = OrderedDict(sorted(o.items()))\n\n            rval = OrderedDict()\n            for key, value in o.items():\n                if not isinstance(key, str):\n                    raise TypeError(\n                        \"Can only use string as keys, you passed \"\n                        f\"type {type(key)} for value {key!s}.\",\n                    )\n                _key = self._serialize_sklearn(key, parent_model)\n                rval[_key] = self._serialize_sklearn(value, parent_model)\n        elif isinstance(o, type):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_type(o)\n        elif isinstance(o, scipy.stats.distributions.rv_frozen):\n            rval = self._serialize_rv_frozen(o)\n        # This only works for user-defined functions (and not even partial).\n        # I think this is exactly what we want here as there shouldn't be any\n        # built-in or functool.partials in a pipeline\n        elif inspect.isfunction(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_function(o)\n        elif self._is_cross_validator(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_cross_validator(o)\n        else:\n            raise TypeError(o, type(o))\n\n        return rval\n\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n        Returns\n        -------\n        List\n        \"\"\"\n        # This can possibly be done by a package such as pyxb, but I could not get\n        # it to work properly.\n        import numpy\n        import scipy\n        import sklearn\n\n        major, minor, micro, _, _ = sys.version_info\n        python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n        sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n        numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n        scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n        return [python_version, sklearn_version, numpy_version, scipy_version]\n\n    def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return \" \".join(self.get_version_information())\n\n    def _is_cross_validator(self, o: Any) -&gt; bool:\n        return isinstance(o, sklearn.model_selection.BaseCrossValidator)\n\n    @classmethod\n    def _is_sklearn_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        sklearn_dependency = isinstance(flow.dependencies, str) and \"sklearn\" in flow.dependencies\n        sklearn_as_external = isinstance(flow.external_version, str) and (\n            flow.external_version.startswith(\"sklearn==\") or \",sklearn==\" in flow.external_version\n        )\n        return sklearn_dependency or sklearn_as_external\n\n    def _get_sklearn_description(self, model: Any, char_lim: int = 1024) -&gt; str:\n        r\"\"\"Fetches the sklearn function docstring for the flow description\n\n        Retrieves the sklearn docstring available and does the following:\n        * If length of docstring &lt;= char_lim, then returns the complete docstring\n        * Else, trims the docstring till it encounters a 'Read more in the :ref:'\n        * Or till it encounters a 'Parameters\\n----------\\n'\n        The final string returned is at most of length char_lim with leading and\n        trailing whitespaces removed.\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters for the 'description' field.\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return \"\"\n        try:\n            # trim till 'Read more'\n            pattern = \"Read more in the :ref:\"\n            index = s.index(pattern)\n            s = s[:index]\n            # trimming docstring to be within char_lim\n            if len(s) &gt; char_lim:\n                s = f\"{s[: char_lim - 3]}...\"\n            return s.strip()\n        except ValueError:\n            logger.warning(\n                \"'Read more' not found in descriptions. \"\n                \"Trying to trim till 'Parameters' if available in docstring.\",\n            )\n        try:\n            # if 'Read more' doesn't exist, trim till 'Parameters'\n            pattern = \"Parameters\"\n            index = s.index(match_format(pattern))\n        except ValueError:\n            # returning full docstring\n            logger.warning(\"'Parameters' not found in docstring. Omitting docstring trimming.\")\n            index = len(s)\n        s = s[:index]\n        # trimming docstring to be within char_lim\n        if len(s) &gt; char_lim:\n            s = f\"{s[: char_lim - 3]}...\"\n        return s.strip()\n\n    def _extract_sklearn_parameter_docstring(self, model) -&gt; None | str:\n        \"\"\"Extracts the part of sklearn docstring containing parameter information\n\n        Fetches the entire docstring and trims just the Parameter section.\n        The assumption is that 'Parameters' is the first section in sklearn docstrings,\n        followed by other sections titled 'Attributes', 'See also', 'Note', 'References',\n        appearing in that order if defined.\n        Returns a None if no section with 'Parameters' can be found in the docstring.\n\n        Parameters\n        ----------\n        model : sklearn model\n\n        Returns\n        -------\n        str, or None\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return None\n        try:\n            index1 = s.index(match_format(\"Parameters\"))\n        except ValueError as e:\n            # when sklearn docstring has no 'Parameters' section\n            logger.warning(\"{} {}\".format(match_format(\"Parameters\"), e))\n            return None\n\n        headings = [\"Attributes\", \"Notes\", \"See also\", \"Note\", \"References\"]\n        for h in headings:\n            try:\n                # to find end of Parameters section\n                index2 = s.index(match_format(h))\n                break\n            except ValueError:\n                logger.warning(f\"{h} not available in docstring\")\n                continue\n        else:\n            # in the case only 'Parameters' exist, trim till end of docstring\n            index2 = len(s)\n        s = s[index1:index2]\n        return s.strip()\n\n    def _extract_sklearn_param_info(self, model, char_lim=1024) -&gt; None | dict:\n        \"\"\"Parses parameter type and description from sklearn dosctring\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters string fields.\n\n        Returns\n        -------\n        Dict, or None\n        \"\"\"\n        docstring = self._extract_sklearn_parameter_docstring(model)\n        if docstring is None:\n            # when sklearn docstring has no 'Parameters' section\n            return None\n\n        n = re.compile(\"[.]*\\n\", flags=IGNORECASE)\n        lines = n.split(docstring)\n        p = re.compile(\"[a-z0-9_ ]+ : [a-z0-9_']+[a-z0-9_ ]*\", flags=IGNORECASE)\n        # The above regular expression is designed to detect sklearn parameter names and type\n        # in the format of [variable_name][space]:[space][type]\n        # The expectation is that the parameter description for this detected parameter will\n        # be all the lines in the docstring till the regex finds another parameter match\n\n        # collecting parameters and their descriptions\n        description = []  # type: List\n        for s in lines:\n            param = p.findall(s)\n            if param != []:\n                # a parameter definition is found by regex\n                # creating placeholder when parameter found which will be a list of strings\n                # string descriptions will be appended in subsequent iterations\n                # till another parameter is found and a new placeholder is created\n                placeholder = [\"\"]  # type: List[str]\n                description.append(placeholder)\n            elif len(description) &gt; 0:  # description=[] means no parameters found yet\n                # appending strings to the placeholder created when parameter found\n                description[-1].append(s)\n        for i in range(len(description)):\n            # concatenating parameter description strings\n            description[i] = \"\\n\".join(description[i]).strip()\n            # limiting all parameter descriptions to accepted OpenML string length\n            if len(description[i]) &gt; char_lim:\n                description[i] = f\"{description[i][: char_lim - 3]}...\"\n\n        # collecting parameters and their types\n        parameter_docs = OrderedDict()\n        matches = p.findall(docstring)\n        for i, param in enumerate(matches):\n            key, value = str(param).split(\":\")\n            parameter_docs[key.strip()] = [value.strip(), description[i]]\n\n        # to avoid KeyError for missing parameters\n        param_list_true = list(model.get_params().keys())\n        param_list_found = list(parameter_docs.keys())\n        for param in list(set(param_list_true) - set(param_list_found)):\n            parameter_docs[param] = [None, None]\n\n        return parameter_docs\n\n    def _serialize_model(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Create an OpenMLFlow.\n\n        Calls `sklearn_to_flow` recursively to properly serialize the\n        parameters to strings and the components (other models) to OpenMLFlows.\n\n        Parameters\n        ----------\n        model : sklearn estimator\n\n        Returns\n        -------\n        OpenMLFlow\n\n        \"\"\"\n        # Get all necessary information about the model objects itself\n        (\n            parameters,\n            parameters_meta_info,\n            subcomponents,\n            subcomponents_explicit,\n        ) = self._extract_information_from_model(model)\n\n        # Check that a component does not occur multiple times in a flow as this\n        # is not supported by OpenML\n        self._check_multiple_occurence_of_component_in_flow(model, subcomponents)\n\n        # Create a flow name, which contains all components in brackets, e.g.:\n        # RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),\n        # StandardScaler,AdaBoostClassifier(DecisionTreeClassifier))\n        class_name = model.__module__ + \".\" + model.__class__.__name__\n\n        # will be part of the name (in brackets)\n        sub_components_names = \"\"\n        for key in subcomponents:\n            name_thing = subcomponents[key]\n            if isinstance(name_thing, OpenMLFlow):\n                name = name_thing.name\n            elif (\n                isinstance(name_thing, str)\n                and subcomponents[key] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                name = name_thing\n            else:\n                raise TypeError(type(subcomponents[key]))\n\n            if key in subcomponents_explicit:\n                sub_components_names += \",\" + key + \"=\" + name\n            else:\n                sub_components_names += \",\" + name\n\n        # slice operation on string in order to get rid of leading comma\n        name = f\"{class_name}({sub_components_names[1:]})\" if sub_components_names else class_name\n        short_name = SklearnExtension.trim_flow_name(name)\n\n        # Get the external versions of all sub-components\n        external_version = self._get_external_version_string(model, subcomponents)\n        dependencies = self._get_dependencies()\n        tags = self._get_tags()\n\n        sklearn_description = self._get_sklearn_description(model)\n        return OpenMLFlow(\n            name=name,\n            class_name=class_name,\n            custom_name=short_name,\n            description=sklearn_description,\n            model=model,\n            components=subcomponents,\n            parameters=parameters,\n            parameters_meta_info=parameters_meta_info,\n            external_version=external_version,\n            tags=tags,\n            extension=self,\n            language=\"English\",\n            dependencies=dependencies,\n        )\n\n    def _get_dependencies(self) -&gt; str:\n        return self._min_dependency_str(sklearn.__version__)  # type: ignore\n\n    def _get_tags(self) -&gt; list[str]:\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        sklearn_version_formatted = sklearn_version.replace(\"==\", \"_\")\n        return [\n            \"openml-python\",\n            \"sklearn\",\n            \"scikit-learn\",\n            \"python\",\n            sklearn_version_formatted,\n            # TODO: add more tags based on the scikit-learn\n            # module a flow is in? For example automatically\n            # annotate a class of sklearn.svm.SVC() with the\n            # tag svm?\n        ]\n\n    def _get_external_version_string(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; str:\n        # Create external version string for a flow, given the model and the\n        # already parsed dictionary of sub_components. Retrieves the external\n        # version of all subcomponents, which themselves already contain all\n        # requirements for their subcomponents. The external version string is a\n        # sorted concatenation of all modules which are present in this run.\n\n        external_versions = set()\n\n        # The model is None if the flow is a placeholder flow such as 'passthrough' or 'drop'\n        if model is not None:\n            model_package_name = model.__module__.split(\".\")[0]\n            module = importlib.import_module(model_package_name)\n            model_package_version_number = module.__version__  # type: ignore\n            external_version = self._format_external_version(\n                model_package_name,\n                model_package_version_number,\n            )\n            external_versions.add(external_version)\n\n        openml_version = self._format_external_version(\"openml\", openml.__version__)\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        external_versions.add(openml_version)\n        external_versions.add(sklearn_version)\n        for visitee in sub_components.values():\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                continue\n            for external_version in visitee.external_version.split(\",\"):\n                external_versions.add(external_version)\n        return \",\".join(sorted(external_versions))\n\n    def _check_multiple_occurence_of_component_in_flow(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; None:\n        to_visit_stack: list[OpenMLFlow] = []\n        to_visit_stack.extend(sub_components.values())\n        known_sub_components: set[str] = set()\n\n        while len(to_visit_stack) &gt; 0:\n            visitee = to_visit_stack.pop()\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                known_sub_components.add(visitee)\n            elif visitee.name in known_sub_components:\n                raise ValueError(\n                    f\"Found a second occurence of component {visitee.name} when \"\n                    f\"trying to serialize {model}.\",\n                )\n            else:\n                known_sub_components.add(visitee.name)\n                to_visit_stack.extend(visitee.components.values())\n\n    def _extract_information_from_model(  # noqa: PLR0915, C901, PLR0912\n        self,\n        model: Any,\n    ) -&gt; tuple[\n        OrderedDict[str, str | None],\n        OrderedDict[str, dict | None],\n        OrderedDict[str, OpenMLFlow],\n        set,\n    ]:\n        # This function contains four \"global\" states and is quite long and\n        # complicated. If it gets to complicated to ensure it's correctness,\n        # it would be best to make it a class with the four \"global\" states being\n        # the class attributes and the if/elif/else in the for-loop calls to\n        # separate class methods\n\n        # stores all entities that should become subcomponents\n        sub_components = OrderedDict()  # type: OrderedDict[str, OpenMLFlow]\n        # stores the keys of all subcomponents that should become\n        sub_components_explicit = set()\n        parameters: OrderedDict[str, str | None] = OrderedDict()\n        parameters_meta_info: OrderedDict[str, dict | None] = OrderedDict()\n        parameters_docs = self._extract_sklearn_param_info(model)\n\n        model_parameters = model.get_params(deep=False)\n        for k, v in sorted(model_parameters.items(), key=lambda t: t[0]):\n            rval = self._serialize_sklearn(v, model)\n\n            def flatten_all(list_):\n                \"\"\"Flattens arbitrary depth lists of lists (e.g. [[1,2],[3,[1]]] -&gt; [1,2,3,1]).\"\"\"\n                for el in list_:\n                    if isinstance(el, (list, tuple)) and len(el) &gt; 0:\n                        yield from flatten_all(el)\n                    else:\n                        yield el\n\n            # In case rval is a list of lists (or tuples), we need to identify two situations:\n            # - sklearn pipeline steps, feature union or base classifiers in voting classifier.\n            #   They look like e.g. [(\"imputer\", Imputer()), (\"classifier\", SVC())]\n            # - a list of lists with simple types (e.g. int or str), such as for an OrdinalEncoder\n            #   where all possible values for each feature are described: [[0,1,2], [1,2,5]]\n            is_non_empty_list_of_lists_with_same_type = (\n                isinstance(rval, (list, tuple))\n                and len(rval) &gt; 0\n                and isinstance(rval[0], (list, tuple))\n                and all(isinstance(rval_i, type(rval[0])) for rval_i in rval)\n            )\n\n            # Check that all list elements are of simple types.\n            nested_list_of_simple_types = (\n                is_non_empty_list_of_lists_with_same_type\n                and all(isinstance(el, SIMPLE_TYPES) for el in flatten_all(rval))\n                and all(\n                    len(rv) in (2, 3) and rv[1] not in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    for rv in rval\n                )\n            )\n\n            if is_non_empty_list_of_lists_with_same_type and not nested_list_of_simple_types:\n                # If a list of lists is identified that include 'non-simple' types (e.g. objects),\n                # we assume they are steps in a pipeline, feature union, or base classifiers in\n                # a voting classifier.\n                parameter_value = []  # type: List\n                reserved_keywords = set(model.get_params(deep=False).keys())\n\n                for sub_component_tuple in rval:\n                    identifier = sub_component_tuple[0]\n                    sub_component = sub_component_tuple[1]\n                    sub_component_type = type(sub_component_tuple)\n                    if not 2 &lt;= len(sub_component_tuple) &lt;= 3:\n                        # length 2 is for {VotingClassifier.estimators,\n                        # Pipeline.steps, FeatureUnion.transformer_list}\n                        # length 3 is for ColumnTransformer\n                        msg = \"Length of tuple of type {} does not match assumptions\".format(\n                            sub_component_type,\n                        )\n                        raise ValueError(msg)\n\n                    if isinstance(sub_component, str):\n                        if sub_component not in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                            msg = (\n                                \"Second item of tuple does not match assumptions. \"\n                                \"If string, can be only 'drop' or 'passthrough' but\"\n                                \"got %s\" % sub_component\n                            )\n                            raise ValueError(msg)\n                    elif sub_component is None:\n                        msg = (\n                            \"Cannot serialize objects of None type. Please use a valid \"\n                            \"placeholder for None. Note that empty sklearn estimators can be \"\n                            \"replaced with 'drop' or 'passthrough'.\"\n                        )\n                        raise ValueError(msg)\n                    elif not isinstance(sub_component, OpenMLFlow):\n                        msg = (\n                            \"Second item of tuple does not match assumptions. \"\n                            \"Expected OpenMLFlow, got %s\" % type(sub_component)\n                        )\n                        raise TypeError(msg)\n\n                    if identifier in reserved_keywords:\n                        parent_model = f\"{model.__module__}.{model.__class__.__name__}\"\n                        msg = \"Found element shadowing official \" \"parameter for {}: {}\".format(\n                            parent_model,\n                            identifier,\n                        )\n                        raise PyOpenMLError(msg)\n\n                    # when deserializing the parameter\n                    sub_components_explicit.add(identifier)\n                    if isinstance(sub_component, str):\n                        external_version = self._get_external_version_string(None, {})\n                        dependencies = self._get_dependencies()\n                        tags = self._get_tags()\n\n                        sub_components[identifier] = OpenMLFlow(\n                            name=sub_component,\n                            description=\"Placeholder flow for scikit-learn's string pipeline \"\n                            \"members\",\n                            components=OrderedDict(),\n                            parameters=OrderedDict(),\n                            parameters_meta_info=OrderedDict(),\n                            external_version=external_version,\n                            tags=tags,\n                            language=\"English\",\n                            dependencies=dependencies,\n                            model=None,\n                        )\n                        component_reference: OrderedDict[str, str | dict] = OrderedDict()\n                        component_reference[\n                            \"oml-python:serialized_object\"\n                        ] = COMPOSITION_STEP_CONSTANT\n                        cr_value: dict[str, Any] = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    else:\n                        sub_components[identifier] = sub_component\n                        component_reference = OrderedDict()\n                        component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                        cr_value = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    parameter_value.append(component_reference)\n\n                # Here (and in the elif and else branch below) are the only\n                # places where we encode a value as json to make sure that all\n                # parameter values still have the same type after\n                # deserialization\n                if isinstance(rval, tuple):\n                    parameter_json = json.dumps(tuple(parameter_value))\n                else:\n                    parameter_json = json.dumps(parameter_value)\n                parameters[k] = parameter_json\n\n            elif isinstance(rval, OpenMLFlow):\n                # A subcomponent, for example the base model in\n                # AdaBoostClassifier\n                sub_components[k] = rval\n                sub_components_explicit.add(k)\n                component_reference = OrderedDict()\n                component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                cr_value = OrderedDict()\n                cr_value[\"key\"] = k\n                cr_value[\"step_name\"] = None\n                component_reference[\"value\"] = cr_value\n                cr = self._serialize_sklearn(component_reference, model)\n                parameters[k] = json.dumps(cr)\n\n            elif not (hasattr(rval, \"__len__\") and len(rval) == 0):\n                rval = json.dumps(rval)\n                parameters[k] = rval\n            # a regular hyperparameter\n            else:\n                parameters[k] = None\n\n            if parameters_docs is not None:\n                data_type, description = parameters_docs[k]\n                parameters_meta_info[k] = OrderedDict(\n                    ((\"description\", description), (\"data_type\", data_type)),\n                )\n            else:\n                parameters_meta_info[k] = OrderedDict(((\"description\", None), (\"data_type\", None)))\n\n        return parameters, parameters_meta_info, sub_components, sub_components_explicit\n\n    def _get_fn_arguments_with_defaults(self, fn_name: Callable) -&gt; tuple[dict, set]:\n        \"\"\"\n        Returns\n        -------\n            i) a dict with all parameter names that have a default value, and\n            ii) a set with all parameter names that do not have a default\n\n        Parameters\n        ----------\n        fn_name : callable\n            The function of which we want to obtain the defaults\n\n        Returns\n        -------\n        params_with_defaults: dict\n            a dict mapping parameter name to the default value\n        params_without_defaults: set\n            a set with all parameters that do not have a default value\n        \"\"\"\n        # parameters with defaults are optional, all others are required.\n        parameters = inspect.signature(fn_name).parameters\n        required_params = set()\n        optional_params = {}\n        for param in parameters:\n            parameter = parameters.get(param)\n            default_val = parameter.default  # type: ignore\n            if default_val is inspect.Signature.empty:\n                required_params.add(param)\n            else:\n                optional_params[param] = default_val\n        return optional_params, required_params\n\n    def _deserialize_model(\n        self,\n        flow: OpenMLFlow,\n        keep_defaults: bool,  # noqa: FBT001\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        logger.info(\"-{} deserialize {}\".format(\"-\" * recursion_depth, flow.name))\n        model_name = flow.class_name\n        self._check_dependencies(flow.dependencies, strict_version=strict_version)\n\n        parameters = flow.parameters\n        components = flow.components\n        parameter_dict: dict[str, Any] = OrderedDict()\n\n        # Do a shallow copy of the components dictionary so we can remove the\n        # components from this copy once we added them into the pipeline. This\n        # allows us to not consider them any more when looping over the\n        # components, but keeping the dictionary of components untouched in the\n        # original components dictionary.\n        components_ = copy.copy(components)\n\n        for name in parameters:\n            value = parameters.get(name)\n            logger.info(\n                \"--{} flow_parameter={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                components=components_,\n                initialize_with_defaults=keep_defaults,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        for name in components:\n            if name in parameter_dict:\n                continue\n            if name not in components_:\n                continue\n            value = components[name]\n            logger.info(\n                \"--{} flow_component={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        if model_name is None and flow.name in SKLEARN_PIPELINE_STRING_COMPONENTS:\n            return flow.name\n\n        assert model_name is not None\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n\n        if keep_defaults:\n            # obtain all params with a default\n            param_defaults, _ = self._get_fn_arguments_with_defaults(model_class.__init__)\n\n            # delete the params that have a default from the dict,\n            # so they get initialized with their default value\n            # except [...]\n            for param in param_defaults:\n                # [...] the ones that also have a key in the components dict.\n                # As OpenML stores different flows for ensembles with different\n                # (base-)components, in OpenML terms, these are not considered\n                # hyperparameters but rather constants (i.e., changing them would\n                # result in a different flow)\n                if param not in components:\n                    del parameter_dict[param]\n\n        return model_class(**parameter_dict)\n\n    def _check_dependencies(\n        self,\n        dependencies: str,\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        if not dependencies:\n            return\n\n        dependencies_list = dependencies.split(\"\\n\")\n        for dependency_string in dependencies_list:\n            match = DEPENDENCIES_PATTERN.match(dependency_string)\n            if not match:\n                raise ValueError(\"Cannot parse dependency %s\" % dependency_string)\n\n            dependency_name = match.group(\"name\")\n            operation = match.group(\"operation\")\n            version = match.group(\"version\")\n\n            module = importlib.import_module(dependency_name)\n            required_version = LooseVersion(version)\n            installed_version = LooseVersion(module.__version__)  # type: ignore\n\n            if operation == \"==\":\n                check = required_version == installed_version\n            elif operation == \"&gt;\":\n                check = installed_version &gt; required_version\n            elif operation == \"&gt;=\":\n                check = (\n                    installed_version &gt; required_version or installed_version == required_version\n                )\n            else:\n                raise NotImplementedError(\"operation '%s' is not supported\" % operation)\n            message = (\n                \"Trying to deserialize a model with dependency \"\n                f\"{dependency_string} not satisfied.\"\n            )\n            if not check:\n                if strict_version:\n                    raise ValueError(message)\n\n                warnings.warn(message, category=UserWarning, stacklevel=2)\n\n    def _serialize_type(self, o: Any) -&gt; OrderedDict[str, str]:\n        mapping = {\n            float: \"float\",\n            np.float32: \"np.float32\",\n            np.float64: \"np.float64\",\n            int: \"int\",\n            np.int32: \"np.int32\",\n            np.int64: \"np.int64\",\n        }\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[float] = \"np.float\"\n            mapping[int] = \"np.int\"\n\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"type\"\n        ret[\"value\"] = mapping[o]\n        return ret\n\n    def _deserialize_type(self, o: str) -&gt; Any:\n        mapping = {\n            \"float\": float,\n            \"np.float32\": np.float32,\n            \"np.float64\": np.float64,\n            \"int\": int,\n            \"np.int32\": np.int32,\n            \"np.int64\": np.int64,\n        }\n\n        # TODO(eddiebergman): Might be able to remove this\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[\"np.float\"] = np.float  # type: ignore # noqa: NPY001\n            mapping[\"np.int\"] = np.int  # type: ignore # noqa: NPY001\n\n        return mapping[o]\n\n    def _serialize_rv_frozen(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        args = o.args\n        kwds = o.kwds\n        a = o.a\n        b = o.b\n        dist = o.dist.__class__.__module__ + \".\" + o.dist.__class__.__name__\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n        ret[\"oml-python:serialized_object\"] = \"rv_frozen\"\n        ret[\"value\"] = OrderedDict(\n            ((\"dist\", dist), (\"a\", a), (\"b\", b), (\"args\", args), (\"kwds\", kwds)),\n        )\n        return ret\n\n    def _deserialize_rv_frozen(self, o: OrderedDict[str, str]) -&gt; Any:\n        args = o[\"args\"]\n        kwds = o[\"kwds\"]\n        a = o[\"a\"]\n        b = o[\"b\"]\n        dist_name = o[\"dist\"]\n\n        module_name = dist_name.rsplit(\".\", 1)\n        try:\n            rv_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        except AttributeError as e:\n            _tb = traceback.format_exc()\n            warnings.warn(\n                f\"Cannot create model {dist_name} for flow. Reason is from error {type(e)}:{e}\"\n                f\"\\nTraceback: {_tb}\",\n                RuntimeWarning,\n                stacklevel=2,\n            )\n            return None\n\n        dist = scipy.stats.distributions.rv_frozen(rv_class(), *args, **kwds)  # type: ignore\n        dist.a = a\n        dist.b = b\n\n        return dist\n\n    def _serialize_function(self, o: Callable) -&gt; OrderedDict[str, str]:\n        name = o.__module__ + \".\" + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"function\"\n        ret[\"value\"] = name\n        return ret\n\n    def _deserialize_function(self, name: str) -&gt; Callable:\n        module_name = name.rsplit(\".\", 1)\n        return getattr(importlib.import_module(module_name[0]), module_name[1])\n\n    def _serialize_cross_validator(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n\n        parameters = OrderedDict()  # type: 'OrderedDict[str, Any]'\n\n        # XXX this is copied from sklearn.model_selection._split\n        cls = o.__class__\n        init = getattr(cls.__init__, \"deprecated_original\", cls.__init__)\n        # Ignore varargs, kw and default values and pop self\n        init_signature = inspect.signature(init)  # type: ignore\n        # Consider the constructor parameters excluding 'self'\n        if init is object.__init__:\n            args = []  # type: List\n        else:\n            args = sorted(\n                [\n                    p.name\n                    for p in init_signature.parameters.values()\n                    if p.name != \"self\" and p.kind != p.VAR_KEYWORD\n                ],\n            )\n\n        for key in args:\n            # We need deprecation warnings to always be on in order to\n            # catch deprecated param values.\n            # This is set in utils/__init__.py but it gets overwritten\n            # when running under python3 somehow.\n            with warnings.catch_warnings(record=True) as w:\n                warnings.simplefilter(\"always\", DeprecationWarning)\n                value = getattr(o, key, None)\n                if w is not None and len(w) and w[0].category == DeprecationWarning:\n                    # if the parameter is deprecated, don't show it\n                    continue\n\n            if not (isinstance(value, Sized) and len(value) == 0):\n                value = json.dumps(value)\n                parameters[key] = value\n            else:\n                parameters[key] = None\n\n        ret[\"oml-python:serialized_object\"] = \"cv_object\"\n        name = o.__module__ + \".\" + o.__class__.__name__\n        value = OrderedDict([(\"name\", name), (\"parameters\", parameters)])\n        ret[\"value\"] = value\n\n        return ret\n\n    def _deserialize_cross_validator(\n        self,\n        value: OrderedDict[str, Any],\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        model_name = value[\"name\"]\n        parameters = value[\"parameters\"]\n\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        for parameter in parameters:\n            parameters[parameter] = self._deserialize_sklearn(\n                parameters[parameter],\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n        return model_class(**parameters)\n\n    def _format_external_version(\n        self,\n        model_package_name: str,\n        model_package_version_number: str,\n    ) -&gt; str:\n        return f\"{model_package_name}=={model_package_version_number}\"\n\n    @staticmethod\n    def _get_parameter_values_recursive(\n        param_grid: dict | list[dict],\n        parameter_name: str,\n    ) -&gt; list[Any]:\n        \"\"\"\n        Returns a list of values for a given hyperparameter, encountered\n        recursively throughout the flow. (e.g., n_jobs can be defined\n        for various flows)\n\n        Parameters\n        ----------\n        param_grid: Union[Dict, List[Dict]]\n            Dict mapping from hyperparameter list to value, to a list of\n            such dicts\n\n        parameter_name: str\n            The hyperparameter that needs to be inspected\n\n        Returns\n        -------\n        List\n            A list of all values of hyperparameters with this name\n        \"\"\"\n        if isinstance(param_grid, dict):\n            return [\n                value\n                for param, value in param_grid.items()\n                if param.split(\"__\")[-1] == parameter_name\n            ]\n\n        if isinstance(param_grid, list):\n            result = []\n            for sub_grid in param_grid:\n                result.extend(\n                    SklearnExtension._get_parameter_values_recursive(sub_grid, parameter_name),\n                )\n            return result\n\n        raise ValueError(\"Param_grid should either be a dict or list of dicts\")\n\n    def _prevent_optimize_n_jobs(self, model):\n        \"\"\"\n        Ensures that HPO classes will not optimize the n_jobs hyperparameter\n\n        Parameters\n        ----------\n        model:\n            The model that will be fitted\n        \"\"\"\n        if self._is_hpo_class(model):\n            if isinstance(model, sklearn.model_selection.GridSearchCV):\n                param_distributions = model.param_grid\n            elif isinstance(model, sklearn.model_selection.RandomizedSearchCV):\n                param_distributions = model.param_distributions\n            else:\n                if hasattr(model, \"param_distributions\"):\n                    param_distributions = model.param_distributions\n                else:\n                    raise AttributeError(\n                        \"Using subclass BaseSearchCV other than \"\n                        \"{GridSearchCV, RandomizedSearchCV}. \"\n                        \"Could not find attribute \"\n                        \"param_distributions.\",\n                    )\n                logger.warning(\n                    \"Warning! Using subclass BaseSearchCV other than \"\n                    \"{GridSearchCV, RandomizedSearchCV}. \"\n                    \"Should implement param check. \",\n                )\n            n_jobs_vals = SklearnExtension._get_parameter_values_recursive(\n                param_distributions,\n                \"n_jobs\",\n            )\n            if len(n_jobs_vals) &gt; 0:\n                raise PyOpenMLError(\n                    \"openml-python should not be used to \" \"optimize the n_jobs parameter.\",\n                )\n\n    ################################################################################################\n    # Methods for performing runs with extension modules\n\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is a scikit-learn estimator.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        o = model\n        return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n\n    def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n        \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n        model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Models that are already seeded will maintain the seed. In this case,\n        only integer seeds are allowed (An exception is raised when a RandomState was used as\n        seed).\n\n        Parameters\n        ----------\n        model : sklearn model\n            The model to be seeded\n        seed : int\n            The seed to initialize the RandomState with. Unseeded subcomponents\n            will be seeded with a random number from the RandomState.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        def _seed_current_object(current_value):\n            if isinstance(current_value, int):  # acceptable behaviour\n                return False\n\n            if isinstance(current_value, np.random.RandomState):\n                raise ValueError(\n                    \"Models initialized with a RandomState object are not \"\n                    \"supported. Please seed with an integer. \",\n                )\n\n            if current_value is not None:\n                raise ValueError(\n                    \"Models should be seeded with int or None (this should never \" \"happen). \",\n                )\n\n            return True\n\n        rs = np.random.RandomState(seed)\n        model_params = model.get_params()\n        random_states = {}\n        for param_name in sorted(model_params):\n            if \"random_state\" in param_name:\n                current_value = model_params[param_name]\n                # important to draw the value at this point (and not in the if\n                # statement) this way we guarantee that if a different set of\n                # subflows is seeded, the same number of the random generator is\n                # used\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    random_states[param_name] = new_value\n\n            # Also seed CV objects!\n            elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n                if not hasattr(model_params[param_name], \"random_state\"):\n                    continue\n\n                current_value = model_params[param_name].random_state\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    model_params[param_name].random_state = new_value\n\n        model.set_params(**random_states)\n        return model\n\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        from sklearn.exceptions import NotFittedError\n        from sklearn.utils.validation import check_is_fitted\n\n        try:\n            # check if model is fitted\n            check_is_fitted(model)\n\n            # Creating random dummy data of arbitrary size\n            dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n            # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n            # robust check that works across sklearn versions and models. Internally, 'predict'\n            # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n            # assured check than explicit calls to 'check_is_fitted'\n            model.predict(dummy_data)\n            # Will reach here if the model was fit on a dataset with 3 features\n            return True\n        except NotFittedError:  # needs to be the first exception to be caught\n            # Model is not fitted, as is required\n            return False\n        except ValueError:\n            # Will reach here if the model was fit on a dataset with more or less than 3 features\n            return True\n\n    def _run_model_on_fold(  # noqa: PLR0915, PLR0913, C901, PLR0912\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame | None = None,\n    ) -&gt; tuple[\n        np.ndarray,\n        pd.DataFrame | None,\n        OrderedDict[str, float],\n        OpenMLRunTrace | None,\n    ]:\n        \"\"\"Run a model on a repeat,fold,subsample triplet of the task and return prediction\n        information.\n\n        Furthermore, it will measure run time measures in case multi-core behaviour allows this.\n        * exact user cpu time will be measured if the number of cores is set (recursive throughout\n        the model) exactly to 1\n        * wall clock time will be measured if the number of cores is set (recursive throughout the\n        model) to any given number (but not when it is set to -1)\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        run_task_get_arff_content. Do not use this function unless you know what you are doing.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        pred_y : np.ndarray\n            Predictions on the training/test set, depending on the task type.\n            For supervised tasks, predictions are on the test set.\n            For unsupervised tasks, predictions are on the training set.\n        proba_y : pd.DataFrame, optional\n            Predicted probabilities for the test set.\n            None, if task is not Classification or Learning Curve prediction.\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : OpenMLRunTrace, optional\n            arff trace object from a fitted model and the trace content obtained by\n            repeatedly calling ``run_model_on_task``\n        \"\"\"\n\n        def _prediction_to_probabilities(\n            y: np.ndarray | list,\n            model_classes: list[Any],\n            class_labels: list[str] | None,\n        ) -&gt; pd.DataFrame:\n            \"\"\"Transforms predicted probabilities to match with OpenML class indices.\n\n            Parameters\n            ----------\n            y : np.ndarray\n                Predicted probabilities (possibly omitting classes if they were not present in the\n                training data).\n            model_classes : list\n                List of classes known_predicted by the model, ordered by their index.\n            class_labels : list\n                List of classes as stored in the task object fetched from server.\n\n            Returns\n            -------\n            pd.DataFrame\n            \"\"\"\n            if class_labels is None:\n                raise ValueError(\"The task has no class labels\")\n\n            if isinstance(y_train, np.ndarray) and isinstance(class_labels[0], str):\n                # mapping (decoding) the predictions to the categories\n                # creating a separate copy to not change the expected pred_y type\n                y = [class_labels[pred] for pred in y]  # list or numpy array of predictions\n\n            # model_classes: sklearn classifier mapping from original array id to\n            # prediction index id\n            if not isinstance(model_classes, list):\n                raise ValueError(\"please convert model classes to list prior to calling this fn\")\n\n            # DataFrame allows more accurate mapping of classes as column names\n            result = pd.DataFrame(\n                0,\n                index=np.arange(len(y)),\n                columns=model_classes,\n                dtype=np.float32,\n            )\n            for obs, prediction in enumerate(y):\n                result.loc[obs, prediction] = 1.0\n            return result\n\n        if isinstance(task, OpenMLSupervisedTask):\n            if y_train is None:\n                raise TypeError(\"argument y_train must not be of type None\")\n            if X_test is None:\n                raise TypeError(\"argument X_test must not be of type None\")\n\n        model_copy = sklearn.base.clone(model, safe=True)\n        # sanity check: prohibit users from optimizing n_jobs\n        self._prevent_optimize_n_jobs(model_copy)\n        # measures and stores runtimes\n        user_defined_measures = OrderedDict()  # type: 'OrderedDict[str, float]'\n        try:\n            # for measuring runtime. Only available since Python 3.3\n            modelfit_start_cputime = time.process_time()\n            modelfit_start_walltime = time.time()\n\n            if isinstance(task, OpenMLSupervisedTask):\n                model_copy.fit(X_train, y_train)  # type: ignore\n            elif isinstance(task, OpenMLClusteringTask):\n                model_copy.fit(X_train)  # type: ignore\n\n            modelfit_dur_cputime = (time.process_time() - modelfit_start_cputime) * 1000\n            modelfit_dur_walltime = (time.time() - modelfit_start_walltime) * 1000\n\n            user_defined_measures[\"usercpu_time_millis_training\"] = modelfit_dur_cputime\n            refit_time = model_copy.refit_time_ * 1000 if hasattr(model_copy, \"refit_time_\") else 0  # type: ignore\n            user_defined_measures[\"wall_clock_time_millis_training\"] = modelfit_dur_walltime\n\n        except AttributeError as e:\n            # typically happens when training a regressor on classification task\n            raise PyOpenMLError(str(e)) from e\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            # search for model classes_ (might differ depending on modeltype)\n            # first, pipelines are a special case (these don't have a classes_\n            # object, but rather borrows it from the last step. We do this manually,\n            # because of the BaseSearch check)\n            if isinstance(model_copy, sklearn.pipeline.Pipeline):\n                used_estimator = model_copy.steps[-1][-1]\n            else:\n                used_estimator = model_copy\n\n            if self._is_hpo_class(used_estimator):\n                model_classes = used_estimator.best_estimator_.classes_\n            else:\n                model_classes = used_estimator.classes_\n\n            if not isinstance(model_classes, list):\n                model_classes = model_classes.tolist()\n\n            # to handle the case when dataset is numpy and categories are encoded\n            # however the class labels stored in task are still categories\n            if isinstance(y_train, np.ndarray) and isinstance(\n                cast(List, task.class_labels)[0],\n                str,\n            ):\n                model_classes = [cast(List[str], task.class_labels)[i] for i in model_classes]\n\n        modelpredict_start_cputime = time.process_time()\n        modelpredict_start_walltime = time.time()\n\n        # In supervised learning this returns the predictions for Y, in clustering\n        # it returns the clusters\n        if isinstance(task, OpenMLSupervisedTask):\n            pred_y = model_copy.predict(X_test)\n        elif isinstance(task, OpenMLClusteringTask):\n            pred_y = model_copy.predict(X_train)\n        else:\n            raise ValueError(task)\n\n        modelpredict_duration_cputime = (time.process_time() - modelpredict_start_cputime) * 1000\n        user_defined_measures[\"usercpu_time_millis_testing\"] = modelpredict_duration_cputime\n        user_defined_measures[\"usercpu_time_millis\"] = (\n            modelfit_dur_cputime + modelpredict_duration_cputime\n        )\n        modelpredict_duration_walltime = (time.time() - modelpredict_start_walltime) * 1000\n        user_defined_measures[\"wall_clock_time_millis_testing\"] = modelpredict_duration_walltime\n        user_defined_measures[\"wall_clock_time_millis\"] = (\n            modelfit_dur_walltime + modelpredict_duration_walltime + refit_time\n        )\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            try:\n                proba_y = model_copy.predict_proba(X_test)\n                proba_y = pd.DataFrame(proba_y, columns=model_classes)  # handles X_test as numpy\n            except AttributeError:  # predict_proba is not available when probability=False\n                proba_y = _prediction_to_probabilities(pred_y, model_classes, task.class_labels)\n\n            if task.class_labels is not None:\n                if proba_y.shape[1] != len(task.class_labels):\n                    # Remap the probabilities in case there was a class missing\n                    # at training time. By default, the classification targets\n                    # are mapped to be zero-based indices to the actual classes.\n                    # Therefore, the model_classes contain the correct indices to\n                    # the correct probability array. Example:\n                    # classes in the dataset: 0, 1, 2, 3, 4, 5\n                    # classes in the training set: 0, 1, 2, 4, 5\n                    # then we need to add a column full of zeros into the probabilities\n                    # for class 3 because the rest of the library expects that the\n                    # probabilities are ordered the same way as the classes are ordered).\n                    message = \"Estimator only predicted for {}/{} classes!\".format(\n                        proba_y.shape[1],\n                        len(task.class_labels),\n                    )\n                    warnings.warn(message, stacklevel=2)\n                    openml.config.logger.warning(message)\n\n                    for _i, col in enumerate(task.class_labels):\n                        # adding missing columns with 0 probability\n                        if col not in model_classes:\n                            proba_y[col] = 0\n                    # We re-order the columns to move possibly added missing columns into place.\n                    proba_y = proba_y[task.class_labels]\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n            if not np.all(set(proba_y.columns) == set(task.class_labels)):\n                missing_cols = list(set(task.class_labels) - set(proba_y.columns))\n                raise ValueError(\"Predicted probabilities missing for the columns: \", missing_cols)\n\n        elif isinstance(task, (OpenMLRegressionTask, OpenMLClusteringTask)):\n            proba_y = None\n        else:\n            raise TypeError(type(task))\n\n        if self._is_hpo_class(model_copy):\n            trace_data = self._extract_trace_data(model_copy, rep_no, fold_no)\n            trace: OpenMLRunTrace | None = self._obtain_arff_trace(\n                model_copy,\n                trace_data,\n            )\n        else:\n            trace = None\n\n        return pred_y, proba_y, user_defined_measures, trace\n\n    def obtain_parameter_values(  # noqa: C901, PLR0915\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n        openml.flows.functions._check_flow_for_server_id(flow)\n\n        def get_flow_dict(_flow):\n            flow_map = {_flow.name: _flow.flow_id}\n            for subflow in _flow.components:\n                flow_map.update(get_flow_dict(_flow.components[subflow]))\n            return flow_map\n\n        def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n            _flow,\n            _flow_dict,\n            component_model,\n            _main_call=False,  # noqa: FBT002\n            main_id=None,\n        ):\n            def is_subcomponent_specification(values):\n                # checks whether the current value can be a specification of\n                # subcomponents, as for example the value for steps parameter\n                # (in Pipeline) or transformers parameter (in\n                # ColumnTransformer).\n                return (\n                    # Specification requires list/tuple of list/tuple with\n                    # at least length 2.\n                    isinstance(values, (tuple, list))\n                    and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                    # And each component needs to be a flow or interpretable string\n                    and all(\n                        isinstance(item[1], openml.flows.OpenMLFlow)\n                        or (\n                            isinstance(item[1], str)\n                            and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        )\n                        for item in values\n                    )\n                )\n\n            # _flow is openml flow object, _param dict maps from flow name to flow\n            # id for the main call, the param dict can be overridden (useful for\n            # unit tests / sentinels) this way, for flows without subflows we do\n            # not have to rely on _flow_dict\n            exp_parameters = set(_flow.parameters)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_parameters = set()\n            else:\n                model_parameters = set(component_model.get_params(deep=False))\n            if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n                flow_params = sorted(exp_parameters)\n                model_params = sorted(model_parameters)\n                raise ValueError(\n                    \"Parameters of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow parameters: \"\n                    f\"{flow_params}\\nmodel parameters: {model_params}\",\n                )\n            exp_components = set(_flow.components)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_components = set()\n            else:\n                _ = set(component_model.get_params(deep=False))\n                model_components = {\n                    mp\n                    for mp in component_model.get_params(deep=True)\n                    if \"__\" not in mp and mp not in _\n                }\n            if len(exp_components.symmetric_difference(model_components)) != 0:\n                is_problem = True\n                if len(exp_components - model_components) &gt; 0:\n                    # If an expected component is not returned as a component by get_params(),\n                    # this means that it is also a parameter -&gt; we need to check that this is\n                    # actually the case\n                    difference = exp_components - model_components\n                    component_in_model_parameters = []\n                    for component in difference:\n                        if component in model_parameters:\n                            component_in_model_parameters.append(True)\n                        else:\n                            component_in_model_parameters.append(False)\n                    is_problem = not all(component_in_model_parameters)\n                if is_problem:\n                    flow_components = sorted(exp_components)\n                    model_components = sorted(model_components)\n                    raise ValueError(\n                        \"Subcomponents of the model do not match the \"\n                        \"parameters expected by the \"\n                        \"flow:\\nexpected flow subcomponents: \"\n                        f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                    )\n\n            _params = []\n            for _param_name in _flow.parameters:\n                _current = OrderedDict()\n                _current[\"oml:name\"] = _param_name\n\n                current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n                # Try to filter out components (a.k.a. subflows) which are\n                # handled further down in the code (by recursively calling\n                # this function)!\n                if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                    continue\n\n                if is_subcomponent_specification(current_param_values):\n                    # complex parameter value, with subcomponents\n                    parsed_values = []\n                    for subcomponent in current_param_values:\n                        # scikit-learn stores usually tuples in the form\n                        # (name (str), subcomponent (mixed), argument\n                        # (mixed)). OpenML replaces the subcomponent by an\n                        # OpenMLFlow object.\n                        if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                            raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                        subcomponent_identifier = subcomponent[0]\n                        subcomponent_flow = subcomponent[1]\n                        if not isinstance(subcomponent_identifier, str):\n                            raise TypeError(\n                                \"Subcomponent identifier should be of type string, \"\n                                f\"but is {type(subcomponent_identifier)}\",\n                            )\n                        if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                            if (\n                                isinstance(subcomponent_flow, str)\n                                and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                            ):\n                                pass\n                            else:\n                                raise TypeError(\n                                    \"Subcomponent flow should be of type flow, but is {}\".format(\n                                        type(subcomponent_flow),\n                                    ),\n                                )\n\n                        current = {\n                            \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                            \"value\": {\n                                \"key\": subcomponent_identifier,\n                                \"step_name\": subcomponent_identifier,\n                            },\n                        }\n                        if len(subcomponent) == 3:\n                            if not isinstance(subcomponent[2], list) and not isinstance(\n                                subcomponent[2],\n                                OrderedDict,\n                            ):\n                                raise TypeError(\n                                    \"Subcomponent argument should be list or OrderedDict\",\n                                )\n                            current[\"value\"][\"argument_1\"] = subcomponent[2]\n                        parsed_values.append(current)\n                    parsed_values = json.dumps(parsed_values)\n                else:\n                    # vanilla parameter value\n                    parsed_values = json.dumps(current_param_values)\n\n                _current[\"oml:value\"] = parsed_values\n                if _main_call:\n                    _current[\"oml:component\"] = main_id\n                else:\n                    _current[\"oml:component\"] = _flow_dict[_flow.name]\n                _params.append(_current)\n\n            for _identifier in _flow.components:\n                subcomponent_model = component_model.get_params()[_identifier]\n                _params.extend(\n                    extract_parameters(\n                        _flow.components[_identifier],\n                        _flow_dict,\n                        subcomponent_model,\n                    ),\n                )\n            return _params\n\n        flow_dict = get_flow_dict(flow)\n        model = model if model is not None else flow.model\n        return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n\n    def _openml_param_name_to_sklearn(\n        self,\n        openml_parameter: openml.setups.OpenMLParameter,\n        flow: OpenMLFlow,\n    ) -&gt; str:\n        \"\"\"\n        Converts the name of an OpenMLParameter into the sklean name, given a flow.\n\n        Parameters\n        ----------\n        openml_parameter: OpenMLParameter\n            The parameter under consideration\n\n        flow: OpenMLFlow\n            The flow that provides context.\n\n        Returns\n        -------\n        sklearn_parameter_name: str\n            The name the parameter will have once used in scikit-learn\n        \"\"\"\n        if not isinstance(openml_parameter, openml.setups.OpenMLParameter):\n            raise ValueError(\"openml_parameter should be an instance of OpenMLParameter\")\n        if not isinstance(flow, OpenMLFlow):\n            raise ValueError(\"flow should be an instance of OpenMLFlow\")\n\n        flow_structure = flow.get_structure(\"name\")\n        if openml_parameter.flow_name not in flow_structure:\n            raise ValueError(\"Obtained OpenMLParameter and OpenMLFlow do not correspond. \")\n        name = openml_parameter.flow_name  # for PEP8\n        return \"__\".join(flow_structure[name] + [openml_parameter.parameter_name])\n\n    ################################################################################################\n    # Methods for hyperparameter optimization\n\n    def _is_hpo_class(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the model performs hyperparameter optimization.\n\n        Used to check whether an optimization trace can be extracted from the model after\n        running it.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.model_selection._search.BaseSearchCV)\n\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n        optimization model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        base_estimator = model.estimator\n        base_estimator.set_params(**trace_iteration.get_parameters())\n        return base_estimator\n\n    def _extract_trace_data(self, model, rep_no, fold_no):\n        \"\"\"Extracts data from a machine learning model's cross-validation results\n        and creates an ARFF (Attribute-Relation File Format) trace.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n        rep_no : int\n            The repetition number.\n        fold_no : int\n            The fold number.\n\n        Returns\n        -------\n        A list of ARFF tracecontent.\n        \"\"\"\n        arff_tracecontent = []\n        for itt_no in range(len(model.cv_results_[\"mean_test_score\"])):\n            # we use the string values for True and False, as it is defined in\n            # this way by the OpenML server\n            selected = \"false\"\n            if itt_no == model.best_index_:\n                selected = \"true\"\n            test_score = model.cv_results_[\"mean_test_score\"][itt_no]\n            arff_line = [rep_no, fold_no, itt_no, test_score, selected]\n            for key in model.cv_results_:\n                if key.startswith(\"param_\"):\n                    value = model.cv_results_[key][itt_no]\n                    # Built-in serializer does not convert all numpy types,\n                    # these methods convert them to built-in types instead.\n                    if isinstance(value, np.generic):\n                        # For scalars it actually returns scalars, not a list\n                        value = value.tolist()\n                    serialized_value = json.dumps(value) if value is not np.ma.masked else np.nan\n                    arff_line.append(serialized_value)\n            arff_tracecontent.append(arff_line)\n        return arff_tracecontent\n\n    def _obtain_arff_trace(\n        self,\n        model: Any,\n        trace_content: list,\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Create arff trace object from a fitted model and the trace content obtained by\n        repeatedly calling ``run_model_on_task``.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n\n        trace_content : List[List]\n            Trace content obtained by ``openml.runs.run_flow_on_task``.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        if not hasattr(model, \"cv_results_\"):\n            raise ValueError(\"model should contain `cv_results_`\")\n\n        # attributes that will be in trace arff, regardless of the model\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n\n        # model dependent attributes for trace arff\n        for key in model.cv_results_:\n            if key.startswith(\"param_\"):\n                # supported types should include all types, including bool,\n                # int float\n                supported_basic_types = (bool, int, float, str)\n                for param_value in model.cv_results_[key]:\n                    if isinstance(param_value, np.generic):\n                        param_value = param_value.tolist()  # noqa: PLW2901\n                    if (\n                        isinstance(param_value, supported_basic_types)\n                        or param_value is None\n                        or param_value is np.ma.masked\n                    ):\n                        # basic string values\n                        type = \"STRING\"  # noqa: A001\n                    elif isinstance(param_value, (list, tuple)) and all(\n                        isinstance(i, int) for i in param_value\n                    ):\n                        # list of integers (usually for selecting features)\n                        # hyperparameter layer_sizes of MLPClassifier\n                        type = \"STRING\"  # noqa: A001\n                    else:\n                        raise TypeError(\"Unsupported param type in param grid: %s\" % key)\n\n                # renamed the attribute param to parameter, as this is a required\n                # OpenML convention - this also guards against name collisions\n                # with the required trace attributes\n                attribute = (PREFIX + key[6:], type)  # type: ignore\n                trace_attributes.append(attribute)\n\n        return OpenMLRunTrace.generate(\n            trace_attributes,\n            trace_content,\n        )\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>classmethod</code>","text":"<p>Check whether a given describes a scikit-learn estimator.</p> <p>This is done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given describes a scikit-learn estimator.\n\n    This is done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return cls._is_sklearn_flow(flow)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>classmethod</code>","text":"<p>Check whether a model is an instance of <code>sklearn.base.BaseEstimator</code>.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return isinstance(model, sklearn.base.BaseEstimator)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    from sklearn.exceptions import NotFittedError\n    from sklearn.utils.validation import check_is_fitted\n\n    try:\n        # check if model is fitted\n        check_is_fitted(model)\n\n        # Creating random dummy data of arbitrary size\n        dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n        # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n        # robust check that works across sklearn versions and models. Internally, 'predict'\n        # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n        # assured check than explicit calls to 'check_is_fitted'\n        model.predict(dummy_data)\n        # Will reach here if the model was fit on a dataset with 3 features\n        return True\n    except NotFittedError:  # needs to be the first exception to be caught\n        # Model is not fitted, as is required\n        return False\n    except ValueError:\n        # Will reach here if the model was fit on a dataset with more or less than 3 features\n        return True\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.create_setup_string","title":"<code>create_setup_string(model)</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n    return \" \".join(self.get_version_information())\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>","text":"<p>Initializes a sklearn model based on a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>mixed</code> <p>the object to deserialize (can be flow object, or any serialized parameter value that is accepted by)</p> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>mixed</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; Any:\n    \"\"\"Initializes a sklearn model based on a flow.\n\n    Parameters\n    ----------\n    flow : mixed\n        the object to deserialize (can be flow object, or any serialized\n        parameter value that is accepted by)\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    mixed\n    \"\"\"\n    return self._deserialize_sklearn(\n        flow,\n        initialize_with_defaults=initialize_with_defaults,\n        strict_version=strict_version,\n    )\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.get_version_information","title":"<code>get_version_information()</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Libraries listed are <code>Python</code>, <code>scikit-learn</code>, <code>numpy</code> and <code>scipy</code>.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n    Returns\n    -------\n    List\n    \"\"\"\n    # This can possibly be done by a package such as pyxb, but I could not get\n    # it to work properly.\n    import numpy\n    import scipy\n    import sklearn\n\n    major, minor, micro, _, _ = sys.version_info\n    python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n    sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n    numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n    scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n    return [python_version, sklearn_version, numpy_version, scipy_version]\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>","text":"<p>Instantiate a <code>base_estimator</code> which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n    optimization model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n    if not self._is_hpo_class(model):\n        raise AssertionError(\n            \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n            % model,\n        )\n    base_estimator = model.estimator\n    base_estimator.set_params(**trace_iteration.get_parameters())\n    return base_estimator\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.is_estimator","title":"<code>is_estimator(model)</code>","text":"<p>Check whether the given model is a scikit-learn estimator.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is a scikit-learn estimator.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    o = model\n    return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.model_to_flow","title":"<code>model_to_flow(model)</code>","text":"<p>Transform a scikit-learn model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n    # Necessary to make pypy not complain about all the different possible return types\n    return self._serialize_sklearn(model)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def obtain_parameter_values(  # noqa: C901, PLR0915\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n    openml.flows.functions._check_flow_for_server_id(flow)\n\n    def get_flow_dict(_flow):\n        flow_map = {_flow.name: _flow.flow_id}\n        for subflow in _flow.components:\n            flow_map.update(get_flow_dict(_flow.components[subflow]))\n        return flow_map\n\n    def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n        _flow,\n        _flow_dict,\n        component_model,\n        _main_call=False,  # noqa: FBT002\n        main_id=None,\n    ):\n        def is_subcomponent_specification(values):\n            # checks whether the current value can be a specification of\n            # subcomponents, as for example the value for steps parameter\n            # (in Pipeline) or transformers parameter (in\n            # ColumnTransformer).\n            return (\n                # Specification requires list/tuple of list/tuple with\n                # at least length 2.\n                isinstance(values, (tuple, list))\n                and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                # And each component needs to be a flow or interpretable string\n                and all(\n                    isinstance(item[1], openml.flows.OpenMLFlow)\n                    or (\n                        isinstance(item[1], str)\n                        and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    )\n                    for item in values\n                )\n            )\n\n        # _flow is openml flow object, _param dict maps from flow name to flow\n        # id for the main call, the param dict can be overridden (useful for\n        # unit tests / sentinels) this way, for flows without subflows we do\n        # not have to rely on _flow_dict\n        exp_parameters = set(_flow.parameters)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_parameters = set()\n        else:\n            model_parameters = set(component_model.get_params(deep=False))\n        if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n            flow_params = sorted(exp_parameters)\n            model_params = sorted(model_parameters)\n            raise ValueError(\n                \"Parameters of the model do not match the \"\n                \"parameters expected by the \"\n                \"flow:\\nexpected flow parameters: \"\n                f\"{flow_params}\\nmodel parameters: {model_params}\",\n            )\n        exp_components = set(_flow.components)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_components = set()\n        else:\n            _ = set(component_model.get_params(deep=False))\n            model_components = {\n                mp\n                for mp in component_model.get_params(deep=True)\n                if \"__\" not in mp and mp not in _\n            }\n        if len(exp_components.symmetric_difference(model_components)) != 0:\n            is_problem = True\n            if len(exp_components - model_components) &gt; 0:\n                # If an expected component is not returned as a component by get_params(),\n                # this means that it is also a parameter -&gt; we need to check that this is\n                # actually the case\n                difference = exp_components - model_components\n                component_in_model_parameters = []\n                for component in difference:\n                    if component in model_parameters:\n                        component_in_model_parameters.append(True)\n                    else:\n                        component_in_model_parameters.append(False)\n                is_problem = not all(component_in_model_parameters)\n            if is_problem:\n                flow_components = sorted(exp_components)\n                model_components = sorted(model_components)\n                raise ValueError(\n                    \"Subcomponents of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow subcomponents: \"\n                    f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                )\n\n        _params = []\n        for _param_name in _flow.parameters:\n            _current = OrderedDict()\n            _current[\"oml:name\"] = _param_name\n\n            current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n            # Try to filter out components (a.k.a. subflows) which are\n            # handled further down in the code (by recursively calling\n            # this function)!\n            if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                continue\n\n            if is_subcomponent_specification(current_param_values):\n                # complex parameter value, with subcomponents\n                parsed_values = []\n                for subcomponent in current_param_values:\n                    # scikit-learn stores usually tuples in the form\n                    # (name (str), subcomponent (mixed), argument\n                    # (mixed)). OpenML replaces the subcomponent by an\n                    # OpenMLFlow object.\n                    if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                        raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                    subcomponent_identifier = subcomponent[0]\n                    subcomponent_flow = subcomponent[1]\n                    if not isinstance(subcomponent_identifier, str):\n                        raise TypeError(\n                            \"Subcomponent identifier should be of type string, \"\n                            f\"but is {type(subcomponent_identifier)}\",\n                        )\n                    if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                        if (\n                            isinstance(subcomponent_flow, str)\n                            and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        ):\n                            pass\n                        else:\n                            raise TypeError(\n                                \"Subcomponent flow should be of type flow, but is {}\".format(\n                                    type(subcomponent_flow),\n                                ),\n                            )\n\n                    current = {\n                        \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                        \"value\": {\n                            \"key\": subcomponent_identifier,\n                            \"step_name\": subcomponent_identifier,\n                        },\n                    }\n                    if len(subcomponent) == 3:\n                        if not isinstance(subcomponent[2], list) and not isinstance(\n                            subcomponent[2],\n                            OrderedDict,\n                        ):\n                            raise TypeError(\n                                \"Subcomponent argument should be list or OrderedDict\",\n                            )\n                        current[\"value\"][\"argument_1\"] = subcomponent[2]\n                    parsed_values.append(current)\n                parsed_values = json.dumps(parsed_values)\n            else:\n                # vanilla parameter value\n                parsed_values = json.dumps(current_param_values)\n\n            _current[\"oml:value\"] = parsed_values\n            if _main_call:\n                _current[\"oml:component\"] = main_id\n            else:\n                _current[\"oml:component\"] = _flow_dict[_flow.name]\n            _params.append(_current)\n\n        for _identifier in _flow.components:\n            subcomponent_model = component_model.get_params()[_identifier]\n            _params.extend(\n                extract_parameters(\n                    _flow.components[_identifier],\n                    _flow_dict,\n                    subcomponent_model,\n                ),\n            )\n        return _params\n\n    flow_dict = get_flow_dict(flow)\n    model = model if model is not None else flow.model\n    return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.seed_model","title":"<code>seed_model(model, seed=None)</code>","text":"<p>Set the random state of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is raised when a RandomState was used as seed).</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> <p>The seed to initialize the RandomState with. Unseeded subcomponents will be seeded with a random number from the RandomState.</p> <code>None</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n    \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n    model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Models that are already seeded will maintain the seed. In this case,\n    only integer seeds are allowed (An exception is raised when a RandomState was used as\n    seed).\n\n    Parameters\n    ----------\n    model : sklearn model\n        The model to be seeded\n    seed : int\n        The seed to initialize the RandomState with. Unseeded subcomponents\n        will be seeded with a random number from the RandomState.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    def _seed_current_object(current_value):\n        if isinstance(current_value, int):  # acceptable behaviour\n            return False\n\n        if isinstance(current_value, np.random.RandomState):\n            raise ValueError(\n                \"Models initialized with a RandomState object are not \"\n                \"supported. Please seed with an integer. \",\n            )\n\n        if current_value is not None:\n            raise ValueError(\n                \"Models should be seeded with int or None (this should never \" \"happen). \",\n            )\n\n        return True\n\n    rs = np.random.RandomState(seed)\n    model_params = model.get_params()\n    random_states = {}\n    for param_name in sorted(model_params):\n        if \"random_state\" in param_name:\n            current_value = model_params[param_name]\n            # important to draw the value at this point (and not in the if\n            # statement) this way we guarantee that if a different set of\n            # subflows is seeded, the same number of the random generator is\n            # used\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                random_states[param_name] = new_value\n\n        # Also seed CV objects!\n        elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n            if not hasattr(model_params[param_name], \"random_state\"):\n                continue\n\n            current_value = model_params[param_name].random_state\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                model_params[param_name].random_state = new_value\n\n    model.set_params(**random_states)\n    return model\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.SklearnExtension.trim_flow_name","title":"<code>trim_flow_name(long_name, extra_trim_length=100, _outer=True)</code>  <code>classmethod</code>","text":"<p>Shorten generated sklearn flow name to at most <code>max_length</code> characters.</p> <p>Flows are assumed to have the following naming structure: <code>(model_selection)? (pipeline)? (steps)+</code> and will be shortened to: <code>sklearn.(selection.)?(pipeline.)?(steps)+</code> e.g. (white spaces and newlines added for readability)</p> <p>.. code ::</p> <pre><code>sklearn.pipeline.Pipeline(\n    columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n        numeric=sklearn.pipeline.Pipeline(\n            imputer=sklearn.preprocessing.imputation.Imputer,\n            standardscaler=sklearn.preprocessing.data.StandardScaler),\n        nominal=sklearn.pipeline.Pipeline(\n            simpleimputer=sklearn.impute.SimpleImputer,\n            onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n    variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n    svc=sklearn.svm.classes.SVC)\n</code></pre> <p>-&gt; <code>sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)</code></p> <p>Parameters:</p> Name Type Description Default <code>long_name</code> <code>str</code> <p>The full flow name generated by the scikit-learn extension.</p> required <code>extra_trim_length</code> <code>int</code> <p>If the trimmed name would exceed <code>extra_trim_length</code> characters, additional trimming of the short name is performed. This reduces the produced short name length. There is no guarantee the end result will not exceed <code>extra_trim_length</code>.</p> <code>100</code> <code>_outer</code> <code>bool(default=True)</code> <p>For internal use only. Specifies if the function is called recursively.</p> <code>True</code> <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef trim_flow_name(  # noqa: C901\n    cls,\n    long_name: str,\n    extra_trim_length: int = 100,\n    _outer: bool = True,  # noqa: FBT001, FBT002\n) -&gt; str:\n    \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n    Flows are assumed to have the following naming structure:\n    ``(model_selection)? (pipeline)? (steps)+``\n    and will be shortened to:\n    ``sklearn.(selection.)?(pipeline.)?(steps)+``\n    e.g. (white spaces and newlines added for readability)\n\n    .. code ::\n\n        sklearn.pipeline.Pipeline(\n            columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                numeric=sklearn.pipeline.Pipeline(\n                    imputer=sklearn.preprocessing.imputation.Imputer,\n                    standardscaler=sklearn.preprocessing.data.StandardScaler),\n                nominal=sklearn.pipeline.Pipeline(\n                    simpleimputer=sklearn.impute.SimpleImputer,\n                    onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n            variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n            svc=sklearn.svm.classes.SVC)\n\n    -&gt;\n    ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n    Parameters\n    ----------\n    long_name : str\n        The full flow name generated by the scikit-learn extension.\n    extra_trim_length: int (default=100)\n        If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n        of the short name is performed. This reduces the produced short name length.\n        There is no guarantee the end result will not exceed `extra_trim_length`.\n    _outer : bool (default=True)\n        For internal use only. Specifies if the function is called recursively.\n\n    Returns\n    -------\n    str\n\n    \"\"\"\n\n    def remove_all_in_parentheses(string: str) -&gt; str:\n        string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        while removals &gt; 0:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        return string\n\n    # Generally, we want to trim all hyperparameters, the exception to that is for model\n    # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n    # So we first trim name of the `estimator` specified in mode selection. For reference, in\n    # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n    # keep it in the final trimmed flow name:\n    # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n    # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n    # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n    # sklearn.tree.tree.DecisionTreeClassifier))\n    if \"sklearn.model_selection\" in long_name:\n        start_index = long_name.index(\"sklearn.model_selection\")\n        estimator_start = (\n            start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n        )\n\n        model_select_boilerplate = long_name[start_index:estimator_start]\n        # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n        model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n        # Now we want to also find and parse the `estimator`, for this we find the closing\n        # parenthesis to the model selection technique:\n        closing_parenthesis_expected = 1\n        for char in long_name[estimator_start:]:\n            if char == \"(\":\n                closing_parenthesis_expected += 1\n            if char == \")\":\n                closing_parenthesis_expected -= 1\n            if closing_parenthesis_expected == 0:\n                break\n\n        _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n        model_select_pipeline = long_name[estimator_start:_end]\n\n        trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n        _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n        model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n        name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n    else:\n        name = long_name\n\n    module_name = long_name.split(\".\")[0]\n    short_name = module_name + \".{}\"\n\n    if name.startswith(\"sklearn.pipeline\"):\n        full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n        pipeline_class = full_pipeline_class.split(\".\")[-1]\n        # We don't want nested pipelines in the short name, so we trim all complicated\n        # subcomponents, i.e. those with parentheses:\n        pipeline = remove_all_in_parentheses(pipeline)\n\n        # then the pipeline steps are formatted e.g.:\n        # step1name=sklearn.submodule.ClassName,step2name...\n        components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n        pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n        if len(short_name.format(pipeline)) &gt; extra_trim_length:\n            pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n    else:\n        # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n        pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n    if not _outer:\n        # Anything from parenthesis in inner calls should not be culled, so we use brackets\n        pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n    else:\n        # Square brackets may be introduced with nested model_selection\n        pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n    return short_name.format(pipeline)\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.cat","title":"<code>cat(X)</code>","text":"<p>Returns True for all categorical columns, False for the rest.</p> <p>This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is required to process each type of columns separately. This function allows transformations meant for categorical columns to access the categorical columns given the dataset as DataFrame.</p> Source code in <code>openml/extensions/sklearn/__init__.py</code> <pre><code>def cat(X: pd.DataFrame) -&gt; pd.Series:\n    \"\"\"Returns True for all categorical columns, False for the rest.\n\n    This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling\n    of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is\n    required to process each type of columns separately.\n    This function allows transformations meant for categorical columns to access the\n    categorical columns given the dataset as DataFrame.\n    \"\"\"\n    if not hasattr(X, \"dtypes\"):\n        raise AttributeError(\"Not a Pandas DataFrame with 'dtypes' as attribute!\")\n    return X.dtypes == \"category\"\n</code></pre>"},{"location":"reference/extensions/sklearn/#openml.extensions.sklearn.cont","title":"<code>cont(X)</code>","text":"<p>Returns True for all non-categorical columns, False for the rest.</p> <p>This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is required to process each type of columns separately. This function allows transformations meant for continuous/numeric columns to access the continuous/numeric columns given the dataset as DataFrame.</p> Source code in <code>openml/extensions/sklearn/__init__.py</code> <pre><code>def cont(X: pd.DataFrame) -&gt; pd.Series:\n    \"\"\"Returns True for all non-categorical columns, False for the rest.\n\n    This is a helper function for OpenML datasets encoded as DataFrames simplifying the handling\n    of mixed data types. To build sklearn models on mixed data types, a ColumnTransformer is\n    required to process each type of columns separately.\n    This function allows transformations meant for continuous/numeric columns to access the\n    continuous/numeric columns given the dataset as DataFrame.\n    \"\"\"\n    if not hasattr(X, \"dtypes\"):\n        raise AttributeError(\"Not a Pandas DataFrame with 'dtypes' as attribute!\")\n    return X.dtypes != \"category\"\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/","title":"extension","text":""},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension","title":"<code>SklearnExtension</code>","text":"<p>               Bases: <code>Extension</code></p> <p>Connect scikit-learn to OpenML-Python. The estimators which use this extension must be scikit-learn compatible, i.e needs to be a subclass of sklearn.base.BaseEstimator\".</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>class SklearnExtension(Extension):\n    \"\"\"Connect scikit-learn to OpenML-Python.\n    The estimators which use this extension must be scikit-learn compatible,\n    i.e needs to be a subclass of sklearn.base.BaseEstimator\".\n    \"\"\"\n\n    ################################################################################################\n    # General setup\n\n    @classmethod\n    def can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        \"\"\"Check whether a given describes a scikit-learn estimator.\n\n        This is done by parsing the ``external_version`` field.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return cls._is_sklearn_flow(flow)\n\n    @classmethod\n    def can_handle_model(cls, model: Any) -&gt; bool:\n        \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.base.BaseEstimator)\n\n    @classmethod\n    def trim_flow_name(  # noqa: C901\n        cls,\n        long_name: str,\n        extra_trim_length: int = 100,\n        _outer: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; str:\n        \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n        Flows are assumed to have the following naming structure:\n        ``(model_selection)? (pipeline)? (steps)+``\n        and will be shortened to:\n        ``sklearn.(selection.)?(pipeline.)?(steps)+``\n        e.g. (white spaces and newlines added for readability)\n\n        .. code ::\n\n            sklearn.pipeline.Pipeline(\n                columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                    numeric=sklearn.pipeline.Pipeline(\n                        imputer=sklearn.preprocessing.imputation.Imputer,\n                        standardscaler=sklearn.preprocessing.data.StandardScaler),\n                    nominal=sklearn.pipeline.Pipeline(\n                        simpleimputer=sklearn.impute.SimpleImputer,\n                        onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n                variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n                svc=sklearn.svm.classes.SVC)\n\n        -&gt;\n        ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n        Parameters\n        ----------\n        long_name : str\n            The full flow name generated by the scikit-learn extension.\n        extra_trim_length: int (default=100)\n            If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n            of the short name is performed. This reduces the produced short name length.\n            There is no guarantee the end result will not exceed `extra_trim_length`.\n        _outer : bool (default=True)\n            For internal use only. Specifies if the function is called recursively.\n\n        Returns\n        -------\n        str\n\n        \"\"\"\n\n        def remove_all_in_parentheses(string: str) -&gt; str:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            while removals &gt; 0:\n                string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n            return string\n\n        # Generally, we want to trim all hyperparameters, the exception to that is for model\n        # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n        # So we first trim name of the `estimator` specified in mode selection. For reference, in\n        # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n        # keep it in the final trimmed flow name:\n        # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n        # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n        # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n        # sklearn.tree.tree.DecisionTreeClassifier))\n        if \"sklearn.model_selection\" in long_name:\n            start_index = long_name.index(\"sklearn.model_selection\")\n            estimator_start = (\n                start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n            )\n\n            model_select_boilerplate = long_name[start_index:estimator_start]\n            # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n            model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n            # Now we want to also find and parse the `estimator`, for this we find the closing\n            # parenthesis to the model selection technique:\n            closing_parenthesis_expected = 1\n            for char in long_name[estimator_start:]:\n                if char == \"(\":\n                    closing_parenthesis_expected += 1\n                if char == \")\":\n                    closing_parenthesis_expected -= 1\n                if closing_parenthesis_expected == 0:\n                    break\n\n            _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n            model_select_pipeline = long_name[estimator_start:_end]\n\n            trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n            _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n            model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n            name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n        else:\n            name = long_name\n\n        module_name = long_name.split(\".\")[0]\n        short_name = module_name + \".{}\"\n\n        if name.startswith(\"sklearn.pipeline\"):\n            full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n            pipeline_class = full_pipeline_class.split(\".\")[-1]\n            # We don't want nested pipelines in the short name, so we trim all complicated\n            # subcomponents, i.e. those with parentheses:\n            pipeline = remove_all_in_parentheses(pipeline)\n\n            # then the pipeline steps are formatted e.g.:\n            # step1name=sklearn.submodule.ClassName,step2name...\n            components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n            pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n            if len(short_name.format(pipeline)) &gt; extra_trim_length:\n                pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n        else:\n            # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n            pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n        if not _outer:\n            # Anything from parenthesis in inner calls should not be culled, so we use brackets\n            pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n        else:\n            # Square brackets may be introduced with nested model_selection\n            pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n        return short_name.format(pipeline)\n\n    @classmethod\n    def _min_dependency_str(cls, sklearn_version: str) -&gt; str:\n        \"\"\"Returns a string containing the minimum dependencies for the sklearn version passed.\n\n        Parameters\n        ----------\n        sklearn_version : str\n            A version string of the xx.xx.xx\n\n        Returns\n        -------\n        str\n        \"\"\"\n        openml_major_version = int(LooseVersion(openml.__version__).version[1])\n        # This explicit check is necessary to support existing entities on the OpenML servers\n        # that used the fixed dependency string (in the else block)\n        if openml_major_version &gt; 11:\n            # OpenML v0.11 onwards supports sklearn&gt;=0.24\n            # assumption: 0.24 onwards sklearn should contain a _min_dependencies.py file with\n            # variables declared for extracting minimum dependency for that version\n            if LooseVersion(sklearn_version) &gt;= \"0.24\":\n                from sklearn import _min_dependencies as _mindep\n\n                dependency_list = {\n                    \"numpy\": f\"{_mindep.NUMPY_MIN_VERSION}\",\n                    \"scipy\": f\"{_mindep.SCIPY_MIN_VERSION}\",\n                    \"joblib\": f\"{_mindep.JOBLIB_MIN_VERSION}\",\n                    \"threadpoolctl\": f\"{_mindep.THREADPOOLCTL_MIN_VERSION}\",\n                }\n            elif LooseVersion(sklearn_version) &gt;= \"0.23\":\n                dependency_list = {\n                    \"numpy\": \"1.13.3\",\n                    \"scipy\": \"0.19.1\",\n                    \"joblib\": \"0.11\",\n                    \"threadpoolctl\": \"2.0.0\",\n                }\n                if LooseVersion(sklearn_version).version[2] == 0:\n                    dependency_list.pop(\"threadpoolctl\")\n            elif LooseVersion(sklearn_version) &gt;= \"0.21\":\n                dependency_list = {\"numpy\": \"1.11.0\", \"scipy\": \"0.17.0\", \"joblib\": \"0.11\"}\n            elif LooseVersion(sklearn_version) &gt;= \"0.19\":\n                dependency_list = {\"numpy\": \"1.8.2\", \"scipy\": \"0.13.3\"}\n            else:\n                dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n        else:\n            # this is INCORRECT for sklearn versions &gt;= 0.19 and &lt; 0.24\n            # given that OpenML has existing flows uploaded with such dependency information,\n            # we change no behaviour for older sklearn version, however from 0.24 onwards\n            # the dependency list will be accurately updated for any flow uploaded to OpenML\n            dependency_list = {\"numpy\": \"1.6.1\", \"scipy\": \"0.9\"}\n\n        sklearn_dep = f\"sklearn=={sklearn_version}\"\n        dep_str = \"\\n\".join([f\"{k}&gt;={v}\" for k, v in dependency_list.items()])\n        return \"\\n\".join([sklearn_dep, dep_str])\n\n    ################################################################################################\n    # Methods for flow serialization and de-serialization\n\n    def flow_to_model(\n        self,\n        flow: OpenMLFlow,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; Any:\n        \"\"\"Initializes a sklearn model based on a flow.\n\n        Parameters\n        ----------\n        flow : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        return self._deserialize_sklearn(\n            flow,\n            initialize_with_defaults=initialize_with_defaults,\n            strict_version=strict_version,\n        )\n\n    def _deserialize_sklearn(  # noqa: PLR0915, C901, PLR0913, PLR0912\n        self,\n        o: Any,\n        components: dict | None = None,\n        initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n        recursion_depth: int = 0,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        \"\"\"Recursive function to deserialize a scikit-learn flow.\n\n        This function inspects an object to deserialize and decides how to do so. This function\n        delegates all work to the respective functions to deserialize special data structures etc.\n        This function works on everything that has been serialized to OpenML: OpenMLFlow,\n        components (which are flows themselves), functions, hyperparameter distributions (for\n        random search) and the actual hyperparameter values themselves.\n\n        Parameters\n        ----------\n        o : mixed\n            the object to deserialize (can be flow object, or any serialized\n            parameter value that is accepted by)\n\n        components : Optional[dict]\n            Components of the current flow being de-serialized. These will not be used when\n            de-serializing the actual flow, but when de-serializing a component reference.\n\n        initialize_with_defaults : bool, optional (default=False)\n            If this flag is set, the hyperparameter values of flows will be\n            ignored and a flow with its defaults is returned.\n\n        recursion_depth : int\n            The depth at which this flow is called, mostly for debugging\n            purposes\n\n        strict_version : bool, default=True\n            Whether to fail if version requirements are not fulfilled.\n\n        Returns\n        -------\n        mixed\n        \"\"\"\n        logger.info(\n            \"-{} flow_to_sklearn START o={}, components={}, init_defaults={}\".format(\n                \"-\" * recursion_depth, o, components, initialize_with_defaults\n            ),\n        )\n        depth_pp = recursion_depth + 1  # shortcut var, depth plus plus\n\n        # First, we need to check whether the presented object is a json string.\n        # JSON strings are used to encoder parameter values. By passing around\n        # json strings for parameters, we make sure that we can flow_to_sklearn\n        # the parameter values to the correct type.\n\n        if isinstance(o, str):\n            with contextlib.suppress(JSONDecodeError):\n                o = json.loads(o)\n\n        if isinstance(o, dict):\n            # Check if the dict encodes a 'special' object, which could not\n            # easily converted into a string, but rather the information to\n            # re-create the object were stored in a dictionary.\n            if \"oml-python:serialized_object\" in o:\n                serialized_type = o[\"oml-python:serialized_object\"]\n                value = o[\"value\"]\n                if serialized_type == \"type\":\n                    rval = self._deserialize_type(value)\n                elif serialized_type == \"rv_frozen\":\n                    rval = self._deserialize_rv_frozen(value)\n                elif serialized_type == \"function\":\n                    rval = self._deserialize_function(value)\n                elif serialized_type in (COMPOSITION_STEP_CONSTANT, COMPONENT_REFERENCE):\n                    if serialized_type == COMPOSITION_STEP_CONSTANT:\n                        pass\n                    elif serialized_type == COMPONENT_REFERENCE:\n                        value = self._deserialize_sklearn(\n                            value,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        )\n                    else:\n                        raise NotImplementedError(serialized_type)\n                    assert components is not None  # Necessary for mypy\n                    step_name = value[\"step_name\"]\n                    key = value[\"key\"]\n                    component = self._deserialize_sklearn(\n                        components[key],\n                        initialize_with_defaults=initialize_with_defaults,\n                        recursion_depth=depth_pp,\n                        strict_version=strict_version,\n                    )\n                    # The component is now added to where it should be used\n                    # later. It should not be passed to the constructor of the\n                    # main flow object.\n                    del components[key]\n                    if step_name is None:\n                        rval = component\n                    elif \"argument_1\" not in value:\n                        rval = (step_name, component)\n                    else:\n                        rval = (step_name, component, value[\"argument_1\"])\n                elif serialized_type == \"cv_object\":\n                    rval = self._deserialize_cross_validator(\n                        value,\n                        recursion_depth=recursion_depth,\n                        strict_version=strict_version,\n                    )\n                else:\n                    raise ValueError(\"Cannot flow_to_sklearn %s\" % serialized_type)\n\n            else:\n                rval = OrderedDict(\n                    (\n                        self._deserialize_sklearn(\n                            o=key,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                        self._deserialize_sklearn(\n                            o=value,\n                            components=components,\n                            initialize_with_defaults=initialize_with_defaults,\n                            recursion_depth=depth_pp,\n                            strict_version=strict_version,\n                        ),\n                    )\n                    for key, value in sorted(o.items())\n                )\n        elif isinstance(o, (list, tuple)):\n            rval = [\n                self._deserialize_sklearn(\n                    o=element,\n                    components=components,\n                    initialize_with_defaults=initialize_with_defaults,\n                    recursion_depth=depth_pp,\n                    strict_version=strict_version,\n                )\n                for element in o\n            ]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, (bool, int, float, str)) or o is None:\n            rval = o\n        elif isinstance(o, OpenMLFlow):\n            if not self._is_sklearn_flow(o):\n                raise ValueError(\"Only sklearn flows can be reinstantiated\")\n            rval = self._deserialize_model(\n                flow=o,\n                keep_defaults=initialize_with_defaults,\n                recursion_depth=recursion_depth,\n                strict_version=strict_version,\n            )\n        else:\n            raise TypeError(o)\n        logger.info(\n            \"-{} flow_to_sklearn END   o={}, rval={}\".format(\"-\" * recursion_depth, o, rval)\n        )\n        return rval\n\n    def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        OpenMLFlow\n        \"\"\"\n        # Necessary to make pypy not complain about all the different possible return types\n        return self._serialize_sklearn(model)\n\n    def _serialize_sklearn(self, o: Any, parent_model: Any | None = None) -&gt; Any:  # noqa: PLR0912, C901\n        rval = None  # type: Any\n\n        # TODO: assert that only on first recursion lvl `parent_model` can be None\n        if self.is_estimator(o):\n            # is the main model or a submodel\n            rval = self._serialize_model(o)\n        elif (\n            isinstance(o, (list, tuple))\n            and len(o) == 2\n            and o[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            and isinstance(parent_model, sklearn.pipeline._BaseComposition)\n        ):\n            rval = o\n        elif isinstance(o, (list, tuple)):\n            # TODO: explain what type of parameter is here\n            rval = [self._serialize_sklearn(element, parent_model) for element in o]\n            if isinstance(o, tuple):\n                rval = tuple(rval)\n        elif isinstance(o, SIMPLE_TYPES) or o is None:\n            if isinstance(o, tuple(SIMPLE_NUMPY_TYPES)):\n                o = o.item()  # type: ignore\n            # base parameter values\n            rval = o\n        elif isinstance(o, dict):\n            # TODO: explain what type of parameter is here\n            if not isinstance(o, OrderedDict):\n                o = OrderedDict(sorted(o.items()))\n\n            rval = OrderedDict()\n            for key, value in o.items():\n                if not isinstance(key, str):\n                    raise TypeError(\n                        \"Can only use string as keys, you passed \"\n                        f\"type {type(key)} for value {key!s}.\",\n                    )\n                _key = self._serialize_sklearn(key, parent_model)\n                rval[_key] = self._serialize_sklearn(value, parent_model)\n        elif isinstance(o, type):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_type(o)\n        elif isinstance(o, scipy.stats.distributions.rv_frozen):\n            rval = self._serialize_rv_frozen(o)\n        # This only works for user-defined functions (and not even partial).\n        # I think this is exactly what we want here as there shouldn't be any\n        # built-in or functool.partials in a pipeline\n        elif inspect.isfunction(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_function(o)\n        elif self._is_cross_validator(o):\n            # TODO: explain what type of parameter is here\n            rval = self._serialize_cross_validator(o)\n        else:\n            raise TypeError(o, type(o))\n\n        return rval\n\n    def get_version_information(self) -&gt; list[str]:\n        \"\"\"List versions of libraries required by the flow.\n\n        Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n        Returns\n        -------\n        List\n        \"\"\"\n        # This can possibly be done by a package such as pyxb, but I could not get\n        # it to work properly.\n        import numpy\n        import scipy\n        import sklearn\n\n        major, minor, micro, _, _ = sys.version_info\n        python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n        sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n        numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n        scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n        return [python_version, sklearn_version, numpy_version, scipy_version]\n\n    def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n        \"\"\"Create a string which can be used to reinstantiate the given model.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        str\n        \"\"\"\n        return \" \".join(self.get_version_information())\n\n    def _is_cross_validator(self, o: Any) -&gt; bool:\n        return isinstance(o, sklearn.model_selection.BaseCrossValidator)\n\n    @classmethod\n    def _is_sklearn_flow(cls, flow: OpenMLFlow) -&gt; bool:\n        sklearn_dependency = isinstance(flow.dependencies, str) and \"sklearn\" in flow.dependencies\n        sklearn_as_external = isinstance(flow.external_version, str) and (\n            flow.external_version.startswith(\"sklearn==\") or \",sklearn==\" in flow.external_version\n        )\n        return sklearn_dependency or sklearn_as_external\n\n    def _get_sklearn_description(self, model: Any, char_lim: int = 1024) -&gt; str:\n        r\"\"\"Fetches the sklearn function docstring for the flow description\n\n        Retrieves the sklearn docstring available and does the following:\n        * If length of docstring &lt;= char_lim, then returns the complete docstring\n        * Else, trims the docstring till it encounters a 'Read more in the :ref:'\n        * Or till it encounters a 'Parameters\\n----------\\n'\n        The final string returned is at most of length char_lim with leading and\n        trailing whitespaces removed.\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters for the 'description' field.\n\n        Returns\n        -------\n        str\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return \"\"\n        try:\n            # trim till 'Read more'\n            pattern = \"Read more in the :ref:\"\n            index = s.index(pattern)\n            s = s[:index]\n            # trimming docstring to be within char_lim\n            if len(s) &gt; char_lim:\n                s = f\"{s[: char_lim - 3]}...\"\n            return s.strip()\n        except ValueError:\n            logger.warning(\n                \"'Read more' not found in descriptions. \"\n                \"Trying to trim till 'Parameters' if available in docstring.\",\n            )\n        try:\n            # if 'Read more' doesn't exist, trim till 'Parameters'\n            pattern = \"Parameters\"\n            index = s.index(match_format(pattern))\n        except ValueError:\n            # returning full docstring\n            logger.warning(\"'Parameters' not found in docstring. Omitting docstring trimming.\")\n            index = len(s)\n        s = s[:index]\n        # trimming docstring to be within char_lim\n        if len(s) &gt; char_lim:\n            s = f\"{s[: char_lim - 3]}...\"\n        return s.strip()\n\n    def _extract_sklearn_parameter_docstring(self, model) -&gt; None | str:\n        \"\"\"Extracts the part of sklearn docstring containing parameter information\n\n        Fetches the entire docstring and trims just the Parameter section.\n        The assumption is that 'Parameters' is the first section in sklearn docstrings,\n        followed by other sections titled 'Attributes', 'See also', 'Note', 'References',\n        appearing in that order if defined.\n        Returns a None if no section with 'Parameters' can be found in the docstring.\n\n        Parameters\n        ----------\n        model : sklearn model\n\n        Returns\n        -------\n        str, or None\n        \"\"\"\n\n        def match_format(s):\n            return \"{}\\n{}\\n\".format(s, len(s) * \"-\")\n\n        s = inspect.getdoc(model)\n        if s is None:\n            return None\n        try:\n            index1 = s.index(match_format(\"Parameters\"))\n        except ValueError as e:\n            # when sklearn docstring has no 'Parameters' section\n            logger.warning(\"{} {}\".format(match_format(\"Parameters\"), e))\n            return None\n\n        headings = [\"Attributes\", \"Notes\", \"See also\", \"Note\", \"References\"]\n        for h in headings:\n            try:\n                # to find end of Parameters section\n                index2 = s.index(match_format(h))\n                break\n            except ValueError:\n                logger.warning(f\"{h} not available in docstring\")\n                continue\n        else:\n            # in the case only 'Parameters' exist, trim till end of docstring\n            index2 = len(s)\n        s = s[index1:index2]\n        return s.strip()\n\n    def _extract_sklearn_param_info(self, model, char_lim=1024) -&gt; None | dict:\n        \"\"\"Parses parameter type and description from sklearn dosctring\n\n        Parameters\n        ----------\n        model : sklearn model\n        char_lim : int\n            Specifying the max length of the returned string.\n            OpenML servers have a constraint of 1024 characters string fields.\n\n        Returns\n        -------\n        Dict, or None\n        \"\"\"\n        docstring = self._extract_sklearn_parameter_docstring(model)\n        if docstring is None:\n            # when sklearn docstring has no 'Parameters' section\n            return None\n\n        n = re.compile(\"[.]*\\n\", flags=IGNORECASE)\n        lines = n.split(docstring)\n        p = re.compile(\"[a-z0-9_ ]+ : [a-z0-9_']+[a-z0-9_ ]*\", flags=IGNORECASE)\n        # The above regular expression is designed to detect sklearn parameter names and type\n        # in the format of [variable_name][space]:[space][type]\n        # The expectation is that the parameter description for this detected parameter will\n        # be all the lines in the docstring till the regex finds another parameter match\n\n        # collecting parameters and their descriptions\n        description = []  # type: List\n        for s in lines:\n            param = p.findall(s)\n            if param != []:\n                # a parameter definition is found by regex\n                # creating placeholder when parameter found which will be a list of strings\n                # string descriptions will be appended in subsequent iterations\n                # till another parameter is found and a new placeholder is created\n                placeholder = [\"\"]  # type: List[str]\n                description.append(placeholder)\n            elif len(description) &gt; 0:  # description=[] means no parameters found yet\n                # appending strings to the placeholder created when parameter found\n                description[-1].append(s)\n        for i in range(len(description)):\n            # concatenating parameter description strings\n            description[i] = \"\\n\".join(description[i]).strip()\n            # limiting all parameter descriptions to accepted OpenML string length\n            if len(description[i]) &gt; char_lim:\n                description[i] = f\"{description[i][: char_lim - 3]}...\"\n\n        # collecting parameters and their types\n        parameter_docs = OrderedDict()\n        matches = p.findall(docstring)\n        for i, param in enumerate(matches):\n            key, value = str(param).split(\":\")\n            parameter_docs[key.strip()] = [value.strip(), description[i]]\n\n        # to avoid KeyError for missing parameters\n        param_list_true = list(model.get_params().keys())\n        param_list_found = list(parameter_docs.keys())\n        for param in list(set(param_list_true) - set(param_list_found)):\n            parameter_docs[param] = [None, None]\n\n        return parameter_docs\n\n    def _serialize_model(self, model: Any) -&gt; OpenMLFlow:\n        \"\"\"Create an OpenMLFlow.\n\n        Calls `sklearn_to_flow` recursively to properly serialize the\n        parameters to strings and the components (other models) to OpenMLFlows.\n\n        Parameters\n        ----------\n        model : sklearn estimator\n\n        Returns\n        -------\n        OpenMLFlow\n\n        \"\"\"\n        # Get all necessary information about the model objects itself\n        (\n            parameters,\n            parameters_meta_info,\n            subcomponents,\n            subcomponents_explicit,\n        ) = self._extract_information_from_model(model)\n\n        # Check that a component does not occur multiple times in a flow as this\n        # is not supported by OpenML\n        self._check_multiple_occurence_of_component_in_flow(model, subcomponents)\n\n        # Create a flow name, which contains all components in brackets, e.g.:\n        # RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),\n        # StandardScaler,AdaBoostClassifier(DecisionTreeClassifier))\n        class_name = model.__module__ + \".\" + model.__class__.__name__\n\n        # will be part of the name (in brackets)\n        sub_components_names = \"\"\n        for key in subcomponents:\n            name_thing = subcomponents[key]\n            if isinstance(name_thing, OpenMLFlow):\n                name = name_thing.name\n            elif (\n                isinstance(name_thing, str)\n                and subcomponents[key] in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                name = name_thing\n            else:\n                raise TypeError(type(subcomponents[key]))\n\n            if key in subcomponents_explicit:\n                sub_components_names += \",\" + key + \"=\" + name\n            else:\n                sub_components_names += \",\" + name\n\n        # slice operation on string in order to get rid of leading comma\n        name = f\"{class_name}({sub_components_names[1:]})\" if sub_components_names else class_name\n        short_name = SklearnExtension.trim_flow_name(name)\n\n        # Get the external versions of all sub-components\n        external_version = self._get_external_version_string(model, subcomponents)\n        dependencies = self._get_dependencies()\n        tags = self._get_tags()\n\n        sklearn_description = self._get_sklearn_description(model)\n        return OpenMLFlow(\n            name=name,\n            class_name=class_name,\n            custom_name=short_name,\n            description=sklearn_description,\n            model=model,\n            components=subcomponents,\n            parameters=parameters,\n            parameters_meta_info=parameters_meta_info,\n            external_version=external_version,\n            tags=tags,\n            extension=self,\n            language=\"English\",\n            dependencies=dependencies,\n        )\n\n    def _get_dependencies(self) -&gt; str:\n        return self._min_dependency_str(sklearn.__version__)  # type: ignore\n\n    def _get_tags(self) -&gt; list[str]:\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        sklearn_version_formatted = sklearn_version.replace(\"==\", \"_\")\n        return [\n            \"openml-python\",\n            \"sklearn\",\n            \"scikit-learn\",\n            \"python\",\n            sklearn_version_formatted,\n            # TODO: add more tags based on the scikit-learn\n            # module a flow is in? For example automatically\n            # annotate a class of sklearn.svm.SVC() with the\n            # tag svm?\n        ]\n\n    def _get_external_version_string(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; str:\n        # Create external version string for a flow, given the model and the\n        # already parsed dictionary of sub_components. Retrieves the external\n        # version of all subcomponents, which themselves already contain all\n        # requirements for their subcomponents. The external version string is a\n        # sorted concatenation of all modules which are present in this run.\n\n        external_versions = set()\n\n        # The model is None if the flow is a placeholder flow such as 'passthrough' or 'drop'\n        if model is not None:\n            model_package_name = model.__module__.split(\".\")[0]\n            module = importlib.import_module(model_package_name)\n            model_package_version_number = module.__version__  # type: ignore\n            external_version = self._format_external_version(\n                model_package_name,\n                model_package_version_number,\n            )\n            external_versions.add(external_version)\n\n        openml_version = self._format_external_version(\"openml\", openml.__version__)\n        sklearn_version = self._format_external_version(\"sklearn\", sklearn.__version__)  # type: ignore\n        external_versions.add(openml_version)\n        external_versions.add(sklearn_version)\n        for visitee in sub_components.values():\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                continue\n            for external_version in visitee.external_version.split(\",\"):\n                external_versions.add(external_version)\n        return \",\".join(sorted(external_versions))\n\n    def _check_multiple_occurence_of_component_in_flow(\n        self,\n        model: Any,\n        sub_components: dict[str, OpenMLFlow],\n    ) -&gt; None:\n        to_visit_stack: list[OpenMLFlow] = []\n        to_visit_stack.extend(sub_components.values())\n        known_sub_components: set[str] = set()\n\n        while len(to_visit_stack) &gt; 0:\n            visitee = to_visit_stack.pop()\n            if isinstance(visitee, str) and visitee in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                known_sub_components.add(visitee)\n            elif visitee.name in known_sub_components:\n                raise ValueError(\n                    f\"Found a second occurence of component {visitee.name} when \"\n                    f\"trying to serialize {model}.\",\n                )\n            else:\n                known_sub_components.add(visitee.name)\n                to_visit_stack.extend(visitee.components.values())\n\n    def _extract_information_from_model(  # noqa: PLR0915, C901, PLR0912\n        self,\n        model: Any,\n    ) -&gt; tuple[\n        OrderedDict[str, str | None],\n        OrderedDict[str, dict | None],\n        OrderedDict[str, OpenMLFlow],\n        set,\n    ]:\n        # This function contains four \"global\" states and is quite long and\n        # complicated. If it gets to complicated to ensure it's correctness,\n        # it would be best to make it a class with the four \"global\" states being\n        # the class attributes and the if/elif/else in the for-loop calls to\n        # separate class methods\n\n        # stores all entities that should become subcomponents\n        sub_components = OrderedDict()  # type: OrderedDict[str, OpenMLFlow]\n        # stores the keys of all subcomponents that should become\n        sub_components_explicit = set()\n        parameters: OrderedDict[str, str | None] = OrderedDict()\n        parameters_meta_info: OrderedDict[str, dict | None] = OrderedDict()\n        parameters_docs = self._extract_sklearn_param_info(model)\n\n        model_parameters = model.get_params(deep=False)\n        for k, v in sorted(model_parameters.items(), key=lambda t: t[0]):\n            rval = self._serialize_sklearn(v, model)\n\n            def flatten_all(list_):\n                \"\"\"Flattens arbitrary depth lists of lists (e.g. [[1,2],[3,[1]]] -&gt; [1,2,3,1]).\"\"\"\n                for el in list_:\n                    if isinstance(el, (list, tuple)) and len(el) &gt; 0:\n                        yield from flatten_all(el)\n                    else:\n                        yield el\n\n            # In case rval is a list of lists (or tuples), we need to identify two situations:\n            # - sklearn pipeline steps, feature union or base classifiers in voting classifier.\n            #   They look like e.g. [(\"imputer\", Imputer()), (\"classifier\", SVC())]\n            # - a list of lists with simple types (e.g. int or str), such as for an OrdinalEncoder\n            #   where all possible values for each feature are described: [[0,1,2], [1,2,5]]\n            is_non_empty_list_of_lists_with_same_type = (\n                isinstance(rval, (list, tuple))\n                and len(rval) &gt; 0\n                and isinstance(rval[0], (list, tuple))\n                and all(isinstance(rval_i, type(rval[0])) for rval_i in rval)\n            )\n\n            # Check that all list elements are of simple types.\n            nested_list_of_simple_types = (\n                is_non_empty_list_of_lists_with_same_type\n                and all(isinstance(el, SIMPLE_TYPES) for el in flatten_all(rval))\n                and all(\n                    len(rv) in (2, 3) and rv[1] not in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    for rv in rval\n                )\n            )\n\n            if is_non_empty_list_of_lists_with_same_type and not nested_list_of_simple_types:\n                # If a list of lists is identified that include 'non-simple' types (e.g. objects),\n                # we assume they are steps in a pipeline, feature union, or base classifiers in\n                # a voting classifier.\n                parameter_value = []  # type: List\n                reserved_keywords = set(model.get_params(deep=False).keys())\n\n                for sub_component_tuple in rval:\n                    identifier = sub_component_tuple[0]\n                    sub_component = sub_component_tuple[1]\n                    sub_component_type = type(sub_component_tuple)\n                    if not 2 &lt;= len(sub_component_tuple) &lt;= 3:\n                        # length 2 is for {VotingClassifier.estimators,\n                        # Pipeline.steps, FeatureUnion.transformer_list}\n                        # length 3 is for ColumnTransformer\n                        msg = \"Length of tuple of type {} does not match assumptions\".format(\n                            sub_component_type,\n                        )\n                        raise ValueError(msg)\n\n                    if isinstance(sub_component, str):\n                        if sub_component not in SKLEARN_PIPELINE_STRING_COMPONENTS:\n                            msg = (\n                                \"Second item of tuple does not match assumptions. \"\n                                \"If string, can be only 'drop' or 'passthrough' but\"\n                                \"got %s\" % sub_component\n                            )\n                            raise ValueError(msg)\n                    elif sub_component is None:\n                        msg = (\n                            \"Cannot serialize objects of None type. Please use a valid \"\n                            \"placeholder for None. Note that empty sklearn estimators can be \"\n                            \"replaced with 'drop' or 'passthrough'.\"\n                        )\n                        raise ValueError(msg)\n                    elif not isinstance(sub_component, OpenMLFlow):\n                        msg = (\n                            \"Second item of tuple does not match assumptions. \"\n                            \"Expected OpenMLFlow, got %s\" % type(sub_component)\n                        )\n                        raise TypeError(msg)\n\n                    if identifier in reserved_keywords:\n                        parent_model = f\"{model.__module__}.{model.__class__.__name__}\"\n                        msg = \"Found element shadowing official \" \"parameter for {}: {}\".format(\n                            parent_model,\n                            identifier,\n                        )\n                        raise PyOpenMLError(msg)\n\n                    # when deserializing the parameter\n                    sub_components_explicit.add(identifier)\n                    if isinstance(sub_component, str):\n                        external_version = self._get_external_version_string(None, {})\n                        dependencies = self._get_dependencies()\n                        tags = self._get_tags()\n\n                        sub_components[identifier] = OpenMLFlow(\n                            name=sub_component,\n                            description=\"Placeholder flow for scikit-learn's string pipeline \"\n                            \"members\",\n                            components=OrderedDict(),\n                            parameters=OrderedDict(),\n                            parameters_meta_info=OrderedDict(),\n                            external_version=external_version,\n                            tags=tags,\n                            language=\"English\",\n                            dependencies=dependencies,\n                            model=None,\n                        )\n                        component_reference: OrderedDict[str, str | dict] = OrderedDict()\n                        component_reference[\n                            \"oml-python:serialized_object\"\n                        ] = COMPOSITION_STEP_CONSTANT\n                        cr_value: dict[str, Any] = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    else:\n                        sub_components[identifier] = sub_component\n                        component_reference = OrderedDict()\n                        component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                        cr_value = OrderedDict()\n                        cr_value[\"key\"] = identifier\n                        cr_value[\"step_name\"] = identifier\n                        if len(sub_component_tuple) == 3:\n                            cr_value[\"argument_1\"] = sub_component_tuple[2]\n                        component_reference[\"value\"] = cr_value\n                    parameter_value.append(component_reference)\n\n                # Here (and in the elif and else branch below) are the only\n                # places where we encode a value as json to make sure that all\n                # parameter values still have the same type after\n                # deserialization\n                if isinstance(rval, tuple):\n                    parameter_json = json.dumps(tuple(parameter_value))\n                else:\n                    parameter_json = json.dumps(parameter_value)\n                parameters[k] = parameter_json\n\n            elif isinstance(rval, OpenMLFlow):\n                # A subcomponent, for example the base model in\n                # AdaBoostClassifier\n                sub_components[k] = rval\n                sub_components_explicit.add(k)\n                component_reference = OrderedDict()\n                component_reference[\"oml-python:serialized_object\"] = COMPONENT_REFERENCE\n                cr_value = OrderedDict()\n                cr_value[\"key\"] = k\n                cr_value[\"step_name\"] = None\n                component_reference[\"value\"] = cr_value\n                cr = self._serialize_sklearn(component_reference, model)\n                parameters[k] = json.dumps(cr)\n\n            elif not (hasattr(rval, \"__len__\") and len(rval) == 0):\n                rval = json.dumps(rval)\n                parameters[k] = rval\n            # a regular hyperparameter\n            else:\n                parameters[k] = None\n\n            if parameters_docs is not None:\n                data_type, description = parameters_docs[k]\n                parameters_meta_info[k] = OrderedDict(\n                    ((\"description\", description), (\"data_type\", data_type)),\n                )\n            else:\n                parameters_meta_info[k] = OrderedDict(((\"description\", None), (\"data_type\", None)))\n\n        return parameters, parameters_meta_info, sub_components, sub_components_explicit\n\n    def _get_fn_arguments_with_defaults(self, fn_name: Callable) -&gt; tuple[dict, set]:\n        \"\"\"\n        Returns\n        -------\n            i) a dict with all parameter names that have a default value, and\n            ii) a set with all parameter names that do not have a default\n\n        Parameters\n        ----------\n        fn_name : callable\n            The function of which we want to obtain the defaults\n\n        Returns\n        -------\n        params_with_defaults: dict\n            a dict mapping parameter name to the default value\n        params_without_defaults: set\n            a set with all parameters that do not have a default value\n        \"\"\"\n        # parameters with defaults are optional, all others are required.\n        parameters = inspect.signature(fn_name).parameters\n        required_params = set()\n        optional_params = {}\n        for param in parameters:\n            parameter = parameters.get(param)\n            default_val = parameter.default  # type: ignore\n            if default_val is inspect.Signature.empty:\n                required_params.add(param)\n            else:\n                optional_params[param] = default_val\n        return optional_params, required_params\n\n    def _deserialize_model(\n        self,\n        flow: OpenMLFlow,\n        keep_defaults: bool,  # noqa: FBT001\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        logger.info(\"-{} deserialize {}\".format(\"-\" * recursion_depth, flow.name))\n        model_name = flow.class_name\n        self._check_dependencies(flow.dependencies, strict_version=strict_version)\n\n        parameters = flow.parameters\n        components = flow.components\n        parameter_dict: dict[str, Any] = OrderedDict()\n\n        # Do a shallow copy of the components dictionary so we can remove the\n        # components from this copy once we added them into the pipeline. This\n        # allows us to not consider them any more when looping over the\n        # components, but keeping the dictionary of components untouched in the\n        # original components dictionary.\n        components_ = copy.copy(components)\n\n        for name in parameters:\n            value = parameters.get(name)\n            logger.info(\n                \"--{} flow_parameter={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                components=components_,\n                initialize_with_defaults=keep_defaults,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        for name in components:\n            if name in parameter_dict:\n                continue\n            if name not in components_:\n                continue\n            value = components[name]\n            logger.info(\n                \"--{} flow_component={}, value={}\".format(\"-\" * recursion_depth, name, value)\n            )\n            rval = self._deserialize_sklearn(\n                value,\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n            parameter_dict[name] = rval\n\n        if model_name is None and flow.name in SKLEARN_PIPELINE_STRING_COMPONENTS:\n            return flow.name\n\n        assert model_name is not None\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n\n        if keep_defaults:\n            # obtain all params with a default\n            param_defaults, _ = self._get_fn_arguments_with_defaults(model_class.__init__)\n\n            # delete the params that have a default from the dict,\n            # so they get initialized with their default value\n            # except [...]\n            for param in param_defaults:\n                # [...] the ones that also have a key in the components dict.\n                # As OpenML stores different flows for ensembles with different\n                # (base-)components, in OpenML terms, these are not considered\n                # hyperparameters but rather constants (i.e., changing them would\n                # result in a different flow)\n                if param not in components:\n                    del parameter_dict[param]\n\n        return model_class(**parameter_dict)\n\n    def _check_dependencies(\n        self,\n        dependencies: str,\n        strict_version: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        if not dependencies:\n            return\n\n        dependencies_list = dependencies.split(\"\\n\")\n        for dependency_string in dependencies_list:\n            match = DEPENDENCIES_PATTERN.match(dependency_string)\n            if not match:\n                raise ValueError(\"Cannot parse dependency %s\" % dependency_string)\n\n            dependency_name = match.group(\"name\")\n            operation = match.group(\"operation\")\n            version = match.group(\"version\")\n\n            module = importlib.import_module(dependency_name)\n            required_version = LooseVersion(version)\n            installed_version = LooseVersion(module.__version__)  # type: ignore\n\n            if operation == \"==\":\n                check = required_version == installed_version\n            elif operation == \"&gt;\":\n                check = installed_version &gt; required_version\n            elif operation == \"&gt;=\":\n                check = (\n                    installed_version &gt; required_version or installed_version == required_version\n                )\n            else:\n                raise NotImplementedError(\"operation '%s' is not supported\" % operation)\n            message = (\n                \"Trying to deserialize a model with dependency \"\n                f\"{dependency_string} not satisfied.\"\n            )\n            if not check:\n                if strict_version:\n                    raise ValueError(message)\n\n                warnings.warn(message, category=UserWarning, stacklevel=2)\n\n    def _serialize_type(self, o: Any) -&gt; OrderedDict[str, str]:\n        mapping = {\n            float: \"float\",\n            np.float32: \"np.float32\",\n            np.float64: \"np.float64\",\n            int: \"int\",\n            np.int32: \"np.int32\",\n            np.int64: \"np.int64\",\n        }\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[float] = \"np.float\"\n            mapping[int] = \"np.int\"\n\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"type\"\n        ret[\"value\"] = mapping[o]\n        return ret\n\n    def _deserialize_type(self, o: str) -&gt; Any:\n        mapping = {\n            \"float\": float,\n            \"np.float32\": np.float32,\n            \"np.float64\": np.float64,\n            \"int\": int,\n            \"np.int32\": np.int32,\n            \"np.int64\": np.int64,\n        }\n\n        # TODO(eddiebergman): Might be able to remove this\n        if LooseVersion(np.__version__) &lt; \"1.24\":\n            mapping[\"np.float\"] = np.float  # type: ignore # noqa: NPY001\n            mapping[\"np.int\"] = np.int  # type: ignore # noqa: NPY001\n\n        return mapping[o]\n\n    def _serialize_rv_frozen(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        args = o.args\n        kwds = o.kwds\n        a = o.a\n        b = o.b\n        dist = o.dist.__class__.__module__ + \".\" + o.dist.__class__.__name__\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n        ret[\"oml-python:serialized_object\"] = \"rv_frozen\"\n        ret[\"value\"] = OrderedDict(\n            ((\"dist\", dist), (\"a\", a), (\"b\", b), (\"args\", args), (\"kwds\", kwds)),\n        )\n        return ret\n\n    def _deserialize_rv_frozen(self, o: OrderedDict[str, str]) -&gt; Any:\n        args = o[\"args\"]\n        kwds = o[\"kwds\"]\n        a = o[\"a\"]\n        b = o[\"b\"]\n        dist_name = o[\"dist\"]\n\n        module_name = dist_name.rsplit(\".\", 1)\n        try:\n            rv_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        except AttributeError as e:\n            _tb = traceback.format_exc()\n            warnings.warn(\n                f\"Cannot create model {dist_name} for flow. Reason is from error {type(e)}:{e}\"\n                f\"\\nTraceback: {_tb}\",\n                RuntimeWarning,\n                stacklevel=2,\n            )\n            return None\n\n        dist = scipy.stats.distributions.rv_frozen(rv_class(), *args, **kwds)  # type: ignore\n        dist.a = a\n        dist.b = b\n\n        return dist\n\n    def _serialize_function(self, o: Callable) -&gt; OrderedDict[str, str]:\n        name = o.__module__ + \".\" + o.__name__\n        ret = OrderedDict()  # type: 'OrderedDict[str, str]'\n        ret[\"oml-python:serialized_object\"] = \"function\"\n        ret[\"value\"] = name\n        return ret\n\n    def _deserialize_function(self, name: str) -&gt; Callable:\n        module_name = name.rsplit(\".\", 1)\n        return getattr(importlib.import_module(module_name[0]), module_name[1])\n\n    def _serialize_cross_validator(self, o: Any) -&gt; OrderedDict[str, str | dict]:\n        ret: OrderedDict[str, str | dict] = OrderedDict()\n\n        parameters = OrderedDict()  # type: 'OrderedDict[str, Any]'\n\n        # XXX this is copied from sklearn.model_selection._split\n        cls = o.__class__\n        init = getattr(cls.__init__, \"deprecated_original\", cls.__init__)\n        # Ignore varargs, kw and default values and pop self\n        init_signature = inspect.signature(init)  # type: ignore\n        # Consider the constructor parameters excluding 'self'\n        if init is object.__init__:\n            args = []  # type: List\n        else:\n            args = sorted(\n                [\n                    p.name\n                    for p in init_signature.parameters.values()\n                    if p.name != \"self\" and p.kind != p.VAR_KEYWORD\n                ],\n            )\n\n        for key in args:\n            # We need deprecation warnings to always be on in order to\n            # catch deprecated param values.\n            # This is set in utils/__init__.py but it gets overwritten\n            # when running under python3 somehow.\n            with warnings.catch_warnings(record=True) as w:\n                warnings.simplefilter(\"always\", DeprecationWarning)\n                value = getattr(o, key, None)\n                if w is not None and len(w) and w[0].category == DeprecationWarning:\n                    # if the parameter is deprecated, don't show it\n                    continue\n\n            if not (isinstance(value, Sized) and len(value) == 0):\n                value = json.dumps(value)\n                parameters[key] = value\n            else:\n                parameters[key] = None\n\n        ret[\"oml-python:serialized_object\"] = \"cv_object\"\n        name = o.__module__ + \".\" + o.__class__.__name__\n        value = OrderedDict([(\"name\", name), (\"parameters\", parameters)])\n        ret[\"value\"] = value\n\n        return ret\n\n    def _deserialize_cross_validator(\n        self,\n        value: OrderedDict[str, Any],\n        recursion_depth: int,\n        strict_version: bool = True,  # noqa: FBT002, FBT001\n    ) -&gt; Any:\n        model_name = value[\"name\"]\n        parameters = value[\"parameters\"]\n\n        module_name = model_name.rsplit(\".\", 1)\n        model_class = getattr(importlib.import_module(module_name[0]), module_name[1])\n        for parameter in parameters:\n            parameters[parameter] = self._deserialize_sklearn(\n                parameters[parameter],\n                recursion_depth=recursion_depth + 1,\n                strict_version=strict_version,\n            )\n        return model_class(**parameters)\n\n    def _format_external_version(\n        self,\n        model_package_name: str,\n        model_package_version_number: str,\n    ) -&gt; str:\n        return f\"{model_package_name}=={model_package_version_number}\"\n\n    @staticmethod\n    def _get_parameter_values_recursive(\n        param_grid: dict | list[dict],\n        parameter_name: str,\n    ) -&gt; list[Any]:\n        \"\"\"\n        Returns a list of values for a given hyperparameter, encountered\n        recursively throughout the flow. (e.g., n_jobs can be defined\n        for various flows)\n\n        Parameters\n        ----------\n        param_grid: Union[Dict, List[Dict]]\n            Dict mapping from hyperparameter list to value, to a list of\n            such dicts\n\n        parameter_name: str\n            The hyperparameter that needs to be inspected\n\n        Returns\n        -------\n        List\n            A list of all values of hyperparameters with this name\n        \"\"\"\n        if isinstance(param_grid, dict):\n            return [\n                value\n                for param, value in param_grid.items()\n                if param.split(\"__\")[-1] == parameter_name\n            ]\n\n        if isinstance(param_grid, list):\n            result = []\n            for sub_grid in param_grid:\n                result.extend(\n                    SklearnExtension._get_parameter_values_recursive(sub_grid, parameter_name),\n                )\n            return result\n\n        raise ValueError(\"Param_grid should either be a dict or list of dicts\")\n\n    def _prevent_optimize_n_jobs(self, model):\n        \"\"\"\n        Ensures that HPO classes will not optimize the n_jobs hyperparameter\n\n        Parameters\n        ----------\n        model:\n            The model that will be fitted\n        \"\"\"\n        if self._is_hpo_class(model):\n            if isinstance(model, sklearn.model_selection.GridSearchCV):\n                param_distributions = model.param_grid\n            elif isinstance(model, sklearn.model_selection.RandomizedSearchCV):\n                param_distributions = model.param_distributions\n            else:\n                if hasattr(model, \"param_distributions\"):\n                    param_distributions = model.param_distributions\n                else:\n                    raise AttributeError(\n                        \"Using subclass BaseSearchCV other than \"\n                        \"{GridSearchCV, RandomizedSearchCV}. \"\n                        \"Could not find attribute \"\n                        \"param_distributions.\",\n                    )\n                logger.warning(\n                    \"Warning! Using subclass BaseSearchCV other than \"\n                    \"{GridSearchCV, RandomizedSearchCV}. \"\n                    \"Should implement param check. \",\n                )\n            n_jobs_vals = SklearnExtension._get_parameter_values_recursive(\n                param_distributions,\n                \"n_jobs\",\n            )\n            if len(n_jobs_vals) &gt; 0:\n                raise PyOpenMLError(\n                    \"openml-python should not be used to \" \"optimize the n_jobs parameter.\",\n                )\n\n    ################################################################################################\n    # Methods for performing runs with extension modules\n\n    def is_estimator(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the given model is a scikit-learn estimator.\n\n        This function is only required for backwards compatibility and will be removed in the\n        near future.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        o = model\n        return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n\n    def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n        \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n        model.\n\n        Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n        Models that are already seeded will maintain the seed. In this case,\n        only integer seeds are allowed (An exception is raised when a RandomState was used as\n        seed).\n\n        Parameters\n        ----------\n        model : sklearn model\n            The model to be seeded\n        seed : int\n            The seed to initialize the RandomState with. Unseeded subcomponents\n            will be seeded with a random number from the RandomState.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n\n        def _seed_current_object(current_value):\n            if isinstance(current_value, int):  # acceptable behaviour\n                return False\n\n            if isinstance(current_value, np.random.RandomState):\n                raise ValueError(\n                    \"Models initialized with a RandomState object are not \"\n                    \"supported. Please seed with an integer. \",\n                )\n\n            if current_value is not None:\n                raise ValueError(\n                    \"Models should be seeded with int or None (this should never \" \"happen). \",\n                )\n\n            return True\n\n        rs = np.random.RandomState(seed)\n        model_params = model.get_params()\n        random_states = {}\n        for param_name in sorted(model_params):\n            if \"random_state\" in param_name:\n                current_value = model_params[param_name]\n                # important to draw the value at this point (and not in the if\n                # statement) this way we guarantee that if a different set of\n                # subflows is seeded, the same number of the random generator is\n                # used\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    random_states[param_name] = new_value\n\n            # Also seed CV objects!\n            elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n                if not hasattr(model_params[param_name], \"random_state\"):\n                    continue\n\n                current_value = model_params[param_name].random_state\n                new_value = rs.randint(0, 2**16)\n                if _seed_current_object(current_value):\n                    model_params[param_name].random_state = new_value\n\n        model.set_params(**random_states)\n        return model\n\n    def check_if_model_fitted(self, model: Any) -&gt; bool:\n        \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        from sklearn.exceptions import NotFittedError\n        from sklearn.utils.validation import check_is_fitted\n\n        try:\n            # check if model is fitted\n            check_is_fitted(model)\n\n            # Creating random dummy data of arbitrary size\n            dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n            # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n            # robust check that works across sklearn versions and models. Internally, 'predict'\n            # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n            # assured check than explicit calls to 'check_is_fitted'\n            model.predict(dummy_data)\n            # Will reach here if the model was fit on a dataset with 3 features\n            return True\n        except NotFittedError:  # needs to be the first exception to be caught\n            # Model is not fitted, as is required\n            return False\n        except ValueError:\n            # Will reach here if the model was fit on a dataset with more or less than 3 features\n            return True\n\n    def _run_model_on_fold(  # noqa: PLR0915, PLR0913, C901, PLR0912\n        self,\n        model: Any,\n        task: OpenMLTask,\n        X_train: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame,\n        rep_no: int,\n        fold_no: int,\n        y_train: np.ndarray | None = None,\n        X_test: np.ndarray | scipy.sparse.spmatrix | pd.DataFrame | None = None,\n    ) -&gt; tuple[\n        np.ndarray,\n        pd.DataFrame | None,\n        OrderedDict[str, float],\n        OpenMLRunTrace | None,\n    ]:\n        \"\"\"Run a model on a repeat,fold,subsample triplet of the task and return prediction\n        information.\n\n        Furthermore, it will measure run time measures in case multi-core behaviour allows this.\n        * exact user cpu time will be measured if the number of cores is set (recursive throughout\n        the model) exactly to 1\n        * wall clock time will be measured if the number of cores is set (recursive throughout the\n        model) to any given number (but not when it is set to -1)\n\n        Returns the data that is necessary to construct the OpenML Run object. Is used by\n        run_task_get_arff_content. Do not use this function unless you know what you are doing.\n\n        Parameters\n        ----------\n        model : Any\n            The UNTRAINED model to run. The model instance will be copied and not altered.\n        task : OpenMLTask\n            The task to run the model on.\n        X_train : array-like\n            Training data for the given repetition and fold.\n        rep_no : int\n            The repeat of the experiment (0-based; in case of 1 time CV, always 0)\n        fold_no : int\n            The fold nr of the experiment (0-based; in case of holdout, always 0)\n        y_train : Optional[np.ndarray] (default=None)\n            Target attributes for supervised tasks. In case of classification, these are integer\n            indices to the potential classes specified by dataset.\n        X_test : Optional, array-like (default=None)\n            Test attributes to test for generalization in supervised tasks.\n\n        Returns\n        -------\n        pred_y : np.ndarray\n            Predictions on the training/test set, depending on the task type.\n            For supervised tasks, predictions are on the test set.\n            For unsupervised tasks, predictions are on the training set.\n        proba_y : pd.DataFrame, optional\n            Predicted probabilities for the test set.\n            None, if task is not Classification or Learning Curve prediction.\n        user_defined_measures : OrderedDict[str, float]\n            User defined measures that were generated on this fold\n        trace : OpenMLRunTrace, optional\n            arff trace object from a fitted model and the trace content obtained by\n            repeatedly calling ``run_model_on_task``\n        \"\"\"\n\n        def _prediction_to_probabilities(\n            y: np.ndarray | list,\n            model_classes: list[Any],\n            class_labels: list[str] | None,\n        ) -&gt; pd.DataFrame:\n            \"\"\"Transforms predicted probabilities to match with OpenML class indices.\n\n            Parameters\n            ----------\n            y : np.ndarray\n                Predicted probabilities (possibly omitting classes if they were not present in the\n                training data).\n            model_classes : list\n                List of classes known_predicted by the model, ordered by their index.\n            class_labels : list\n                List of classes as stored in the task object fetched from server.\n\n            Returns\n            -------\n            pd.DataFrame\n            \"\"\"\n            if class_labels is None:\n                raise ValueError(\"The task has no class labels\")\n\n            if isinstance(y_train, np.ndarray) and isinstance(class_labels[0], str):\n                # mapping (decoding) the predictions to the categories\n                # creating a separate copy to not change the expected pred_y type\n                y = [class_labels[pred] for pred in y]  # list or numpy array of predictions\n\n            # model_classes: sklearn classifier mapping from original array id to\n            # prediction index id\n            if not isinstance(model_classes, list):\n                raise ValueError(\"please convert model classes to list prior to calling this fn\")\n\n            # DataFrame allows more accurate mapping of classes as column names\n            result = pd.DataFrame(\n                0,\n                index=np.arange(len(y)),\n                columns=model_classes,\n                dtype=np.float32,\n            )\n            for obs, prediction in enumerate(y):\n                result.loc[obs, prediction] = 1.0\n            return result\n\n        if isinstance(task, OpenMLSupervisedTask):\n            if y_train is None:\n                raise TypeError(\"argument y_train must not be of type None\")\n            if X_test is None:\n                raise TypeError(\"argument X_test must not be of type None\")\n\n        model_copy = sklearn.base.clone(model, safe=True)\n        # sanity check: prohibit users from optimizing n_jobs\n        self._prevent_optimize_n_jobs(model_copy)\n        # measures and stores runtimes\n        user_defined_measures = OrderedDict()  # type: 'OrderedDict[str, float]'\n        try:\n            # for measuring runtime. Only available since Python 3.3\n            modelfit_start_cputime = time.process_time()\n            modelfit_start_walltime = time.time()\n\n            if isinstance(task, OpenMLSupervisedTask):\n                model_copy.fit(X_train, y_train)  # type: ignore\n            elif isinstance(task, OpenMLClusteringTask):\n                model_copy.fit(X_train)  # type: ignore\n\n            modelfit_dur_cputime = (time.process_time() - modelfit_start_cputime) * 1000\n            modelfit_dur_walltime = (time.time() - modelfit_start_walltime) * 1000\n\n            user_defined_measures[\"usercpu_time_millis_training\"] = modelfit_dur_cputime\n            refit_time = model_copy.refit_time_ * 1000 if hasattr(model_copy, \"refit_time_\") else 0  # type: ignore\n            user_defined_measures[\"wall_clock_time_millis_training\"] = modelfit_dur_walltime\n\n        except AttributeError as e:\n            # typically happens when training a regressor on classification task\n            raise PyOpenMLError(str(e)) from e\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            # search for model classes_ (might differ depending on modeltype)\n            # first, pipelines are a special case (these don't have a classes_\n            # object, but rather borrows it from the last step. We do this manually,\n            # because of the BaseSearch check)\n            if isinstance(model_copy, sklearn.pipeline.Pipeline):\n                used_estimator = model_copy.steps[-1][-1]\n            else:\n                used_estimator = model_copy\n\n            if self._is_hpo_class(used_estimator):\n                model_classes = used_estimator.best_estimator_.classes_\n            else:\n                model_classes = used_estimator.classes_\n\n            if not isinstance(model_classes, list):\n                model_classes = model_classes.tolist()\n\n            # to handle the case when dataset is numpy and categories are encoded\n            # however the class labels stored in task are still categories\n            if isinstance(y_train, np.ndarray) and isinstance(\n                cast(List, task.class_labels)[0],\n                str,\n            ):\n                model_classes = [cast(List[str], task.class_labels)[i] for i in model_classes]\n\n        modelpredict_start_cputime = time.process_time()\n        modelpredict_start_walltime = time.time()\n\n        # In supervised learning this returns the predictions for Y, in clustering\n        # it returns the clusters\n        if isinstance(task, OpenMLSupervisedTask):\n            pred_y = model_copy.predict(X_test)\n        elif isinstance(task, OpenMLClusteringTask):\n            pred_y = model_copy.predict(X_train)\n        else:\n            raise ValueError(task)\n\n        modelpredict_duration_cputime = (time.process_time() - modelpredict_start_cputime) * 1000\n        user_defined_measures[\"usercpu_time_millis_testing\"] = modelpredict_duration_cputime\n        user_defined_measures[\"usercpu_time_millis\"] = (\n            modelfit_dur_cputime + modelpredict_duration_cputime\n        )\n        modelpredict_duration_walltime = (time.time() - modelpredict_start_walltime) * 1000\n        user_defined_measures[\"wall_clock_time_millis_testing\"] = modelpredict_duration_walltime\n        user_defined_measures[\"wall_clock_time_millis\"] = (\n            modelfit_dur_walltime + modelpredict_duration_walltime + refit_time\n        )\n\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            try:\n                proba_y = model_copy.predict_proba(X_test)\n                proba_y = pd.DataFrame(proba_y, columns=model_classes)  # handles X_test as numpy\n            except AttributeError:  # predict_proba is not available when probability=False\n                proba_y = _prediction_to_probabilities(pred_y, model_classes, task.class_labels)\n\n            if task.class_labels is not None:\n                if proba_y.shape[1] != len(task.class_labels):\n                    # Remap the probabilities in case there was a class missing\n                    # at training time. By default, the classification targets\n                    # are mapped to be zero-based indices to the actual classes.\n                    # Therefore, the model_classes contain the correct indices to\n                    # the correct probability array. Example:\n                    # classes in the dataset: 0, 1, 2, 3, 4, 5\n                    # classes in the training set: 0, 1, 2, 4, 5\n                    # then we need to add a column full of zeros into the probabilities\n                    # for class 3 because the rest of the library expects that the\n                    # probabilities are ordered the same way as the classes are ordered).\n                    message = \"Estimator only predicted for {}/{} classes!\".format(\n                        proba_y.shape[1],\n                        len(task.class_labels),\n                    )\n                    warnings.warn(message, stacklevel=2)\n                    openml.config.logger.warning(message)\n\n                    for _i, col in enumerate(task.class_labels):\n                        # adding missing columns with 0 probability\n                        if col not in model_classes:\n                            proba_y[col] = 0\n                    # We re-order the columns to move possibly added missing columns into place.\n                    proba_y = proba_y[task.class_labels]\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n            if not np.all(set(proba_y.columns) == set(task.class_labels)):\n                missing_cols = list(set(task.class_labels) - set(proba_y.columns))\n                raise ValueError(\"Predicted probabilities missing for the columns: \", missing_cols)\n\n        elif isinstance(task, (OpenMLRegressionTask, OpenMLClusteringTask)):\n            proba_y = None\n        else:\n            raise TypeError(type(task))\n\n        if self._is_hpo_class(model_copy):\n            trace_data = self._extract_trace_data(model_copy, rep_no, fold_no)\n            trace: OpenMLRunTrace | None = self._obtain_arff_trace(\n                model_copy,\n                trace_data,\n            )\n        else:\n            trace = None\n\n        return pred_y, proba_y, user_defined_measures, trace\n\n    def obtain_parameter_values(  # noqa: C901, PLR0915\n        self,\n        flow: OpenMLFlow,\n        model: Any = None,\n    ) -&gt; list[dict[str, Any]]:\n        \"\"\"Extracts all parameter settings required for the flow from the model.\n\n        If no explicit model is provided, the parameters will be extracted from `flow.model`\n        instead.\n\n        Parameters\n        ----------\n        flow : OpenMLFlow\n            OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n        model: Any, optional (default=None)\n            The model from which to obtain the parameter values. Must match the flow signature.\n            If None, use the model specified in ``OpenMLFlow.model``.\n\n        Returns\n        -------\n        list\n            A list of dicts, where each dict has the following entries:\n            - ``oml:name`` : str: The OpenML parameter name\n            - ``oml:value`` : mixed: A representation of the parameter value\n            - ``oml:component`` : int: flow id to which the parameter belongs\n        \"\"\"\n        openml.flows.functions._check_flow_for_server_id(flow)\n\n        def get_flow_dict(_flow):\n            flow_map = {_flow.name: _flow.flow_id}\n            for subflow in _flow.components:\n                flow_map.update(get_flow_dict(_flow.components[subflow]))\n            return flow_map\n\n        def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n            _flow,\n            _flow_dict,\n            component_model,\n            _main_call=False,  # noqa: FBT002\n            main_id=None,\n        ):\n            def is_subcomponent_specification(values):\n                # checks whether the current value can be a specification of\n                # subcomponents, as for example the value for steps parameter\n                # (in Pipeline) or transformers parameter (in\n                # ColumnTransformer).\n                return (\n                    # Specification requires list/tuple of list/tuple with\n                    # at least length 2.\n                    isinstance(values, (tuple, list))\n                    and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                    # And each component needs to be a flow or interpretable string\n                    and all(\n                        isinstance(item[1], openml.flows.OpenMLFlow)\n                        or (\n                            isinstance(item[1], str)\n                            and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        )\n                        for item in values\n                    )\n                )\n\n            # _flow is openml flow object, _param dict maps from flow name to flow\n            # id for the main call, the param dict can be overridden (useful for\n            # unit tests / sentinels) this way, for flows without subflows we do\n            # not have to rely on _flow_dict\n            exp_parameters = set(_flow.parameters)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_parameters = set()\n            else:\n                model_parameters = set(component_model.get_params(deep=False))\n            if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n                flow_params = sorted(exp_parameters)\n                model_params = sorted(model_parameters)\n                raise ValueError(\n                    \"Parameters of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow parameters: \"\n                    f\"{flow_params}\\nmodel parameters: {model_params}\",\n                )\n            exp_components = set(_flow.components)\n            if (\n                isinstance(component_model, str)\n                and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n            ):\n                model_components = set()\n            else:\n                _ = set(component_model.get_params(deep=False))\n                model_components = {\n                    mp\n                    for mp in component_model.get_params(deep=True)\n                    if \"__\" not in mp and mp not in _\n                }\n            if len(exp_components.symmetric_difference(model_components)) != 0:\n                is_problem = True\n                if len(exp_components - model_components) &gt; 0:\n                    # If an expected component is not returned as a component by get_params(),\n                    # this means that it is also a parameter -&gt; we need to check that this is\n                    # actually the case\n                    difference = exp_components - model_components\n                    component_in_model_parameters = []\n                    for component in difference:\n                        if component in model_parameters:\n                            component_in_model_parameters.append(True)\n                        else:\n                            component_in_model_parameters.append(False)\n                    is_problem = not all(component_in_model_parameters)\n                if is_problem:\n                    flow_components = sorted(exp_components)\n                    model_components = sorted(model_components)\n                    raise ValueError(\n                        \"Subcomponents of the model do not match the \"\n                        \"parameters expected by the \"\n                        \"flow:\\nexpected flow subcomponents: \"\n                        f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                    )\n\n            _params = []\n            for _param_name in _flow.parameters:\n                _current = OrderedDict()\n                _current[\"oml:name\"] = _param_name\n\n                current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n                # Try to filter out components (a.k.a. subflows) which are\n                # handled further down in the code (by recursively calling\n                # this function)!\n                if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                    continue\n\n                if is_subcomponent_specification(current_param_values):\n                    # complex parameter value, with subcomponents\n                    parsed_values = []\n                    for subcomponent in current_param_values:\n                        # scikit-learn stores usually tuples in the form\n                        # (name (str), subcomponent (mixed), argument\n                        # (mixed)). OpenML replaces the subcomponent by an\n                        # OpenMLFlow object.\n                        if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                            raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                        subcomponent_identifier = subcomponent[0]\n                        subcomponent_flow = subcomponent[1]\n                        if not isinstance(subcomponent_identifier, str):\n                            raise TypeError(\n                                \"Subcomponent identifier should be of type string, \"\n                                f\"but is {type(subcomponent_identifier)}\",\n                            )\n                        if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                            if (\n                                isinstance(subcomponent_flow, str)\n                                and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                            ):\n                                pass\n                            else:\n                                raise TypeError(\n                                    \"Subcomponent flow should be of type flow, but is {}\".format(\n                                        type(subcomponent_flow),\n                                    ),\n                                )\n\n                        current = {\n                            \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                            \"value\": {\n                                \"key\": subcomponent_identifier,\n                                \"step_name\": subcomponent_identifier,\n                            },\n                        }\n                        if len(subcomponent) == 3:\n                            if not isinstance(subcomponent[2], list) and not isinstance(\n                                subcomponent[2],\n                                OrderedDict,\n                            ):\n                                raise TypeError(\n                                    \"Subcomponent argument should be list or OrderedDict\",\n                                )\n                            current[\"value\"][\"argument_1\"] = subcomponent[2]\n                        parsed_values.append(current)\n                    parsed_values = json.dumps(parsed_values)\n                else:\n                    # vanilla parameter value\n                    parsed_values = json.dumps(current_param_values)\n\n                _current[\"oml:value\"] = parsed_values\n                if _main_call:\n                    _current[\"oml:component\"] = main_id\n                else:\n                    _current[\"oml:component\"] = _flow_dict[_flow.name]\n                _params.append(_current)\n\n            for _identifier in _flow.components:\n                subcomponent_model = component_model.get_params()[_identifier]\n                _params.extend(\n                    extract_parameters(\n                        _flow.components[_identifier],\n                        _flow_dict,\n                        subcomponent_model,\n                    ),\n                )\n            return _params\n\n        flow_dict = get_flow_dict(flow)\n        model = model if model is not None else flow.model\n        return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n\n    def _openml_param_name_to_sklearn(\n        self,\n        openml_parameter: openml.setups.OpenMLParameter,\n        flow: OpenMLFlow,\n    ) -&gt; str:\n        \"\"\"\n        Converts the name of an OpenMLParameter into the sklean name, given a flow.\n\n        Parameters\n        ----------\n        openml_parameter: OpenMLParameter\n            The parameter under consideration\n\n        flow: OpenMLFlow\n            The flow that provides context.\n\n        Returns\n        -------\n        sklearn_parameter_name: str\n            The name the parameter will have once used in scikit-learn\n        \"\"\"\n        if not isinstance(openml_parameter, openml.setups.OpenMLParameter):\n            raise ValueError(\"openml_parameter should be an instance of OpenMLParameter\")\n        if not isinstance(flow, OpenMLFlow):\n            raise ValueError(\"flow should be an instance of OpenMLFlow\")\n\n        flow_structure = flow.get_structure(\"name\")\n        if openml_parameter.flow_name not in flow_structure:\n            raise ValueError(\"Obtained OpenMLParameter and OpenMLFlow do not correspond. \")\n        name = openml_parameter.flow_name  # for PEP8\n        return \"__\".join(flow_structure[name] + [openml_parameter.parameter_name])\n\n    ################################################################################################\n    # Methods for hyperparameter optimization\n\n    def _is_hpo_class(self, model: Any) -&gt; bool:\n        \"\"\"Check whether the model performs hyperparameter optimization.\n\n        Used to check whether an optimization trace can be extracted from the model after\n        running it.\n\n        Parameters\n        ----------\n        model : Any\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return isinstance(model, sklearn.model_selection._search.BaseSearchCV)\n\n    def instantiate_model_from_hpo_class(\n        self,\n        model: Any,\n        trace_iteration: OpenMLTraceIteration,\n    ) -&gt; Any:\n        \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n        optimization model.\n\n        Parameters\n        ----------\n        model : Any\n            A hyperparameter optimization model which defines the model to be instantiated.\n        trace_iteration : OpenMLTraceIteration\n            Describing the hyperparameter settings to instantiate.\n\n        Returns\n        -------\n        Any\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        base_estimator = model.estimator\n        base_estimator.set_params(**trace_iteration.get_parameters())\n        return base_estimator\n\n    def _extract_trace_data(self, model, rep_no, fold_no):\n        \"\"\"Extracts data from a machine learning model's cross-validation results\n        and creates an ARFF (Attribute-Relation File Format) trace.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n        rep_no : int\n            The repetition number.\n        fold_no : int\n            The fold number.\n\n        Returns\n        -------\n        A list of ARFF tracecontent.\n        \"\"\"\n        arff_tracecontent = []\n        for itt_no in range(len(model.cv_results_[\"mean_test_score\"])):\n            # we use the string values for True and False, as it is defined in\n            # this way by the OpenML server\n            selected = \"false\"\n            if itt_no == model.best_index_:\n                selected = \"true\"\n            test_score = model.cv_results_[\"mean_test_score\"][itt_no]\n            arff_line = [rep_no, fold_no, itt_no, test_score, selected]\n            for key in model.cv_results_:\n                if key.startswith(\"param_\"):\n                    value = model.cv_results_[key][itt_no]\n                    # Built-in serializer does not convert all numpy types,\n                    # these methods convert them to built-in types instead.\n                    if isinstance(value, np.generic):\n                        # For scalars it actually returns scalars, not a list\n                        value = value.tolist()\n                    serialized_value = json.dumps(value) if value is not np.ma.masked else np.nan\n                    arff_line.append(serialized_value)\n            arff_tracecontent.append(arff_line)\n        return arff_tracecontent\n\n    def _obtain_arff_trace(\n        self,\n        model: Any,\n        trace_content: list,\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Create arff trace object from a fitted model and the trace content obtained by\n        repeatedly calling ``run_model_on_task``.\n\n        Parameters\n        ----------\n        model : Any\n            A fitted hyperparameter optimization model.\n\n        trace_content : List[List]\n            Trace content obtained by ``openml.runs.run_flow_on_task``.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if not self._is_hpo_class(model):\n            raise AssertionError(\n                \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n                % model,\n            )\n        if not hasattr(model, \"cv_results_\"):\n            raise ValueError(\"model should contain `cv_results_`\")\n\n        # attributes that will be in trace arff, regardless of the model\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n\n        # model dependent attributes for trace arff\n        for key in model.cv_results_:\n            if key.startswith(\"param_\"):\n                # supported types should include all types, including bool,\n                # int float\n                supported_basic_types = (bool, int, float, str)\n                for param_value in model.cv_results_[key]:\n                    if isinstance(param_value, np.generic):\n                        param_value = param_value.tolist()  # noqa: PLW2901\n                    if (\n                        isinstance(param_value, supported_basic_types)\n                        or param_value is None\n                        or param_value is np.ma.masked\n                    ):\n                        # basic string values\n                        type = \"STRING\"  # noqa: A001\n                    elif isinstance(param_value, (list, tuple)) and all(\n                        isinstance(i, int) for i in param_value\n                    ):\n                        # list of integers (usually for selecting features)\n                        # hyperparameter layer_sizes of MLPClassifier\n                        type = \"STRING\"  # noqa: A001\n                    else:\n                        raise TypeError(\"Unsupported param type in param grid: %s\" % key)\n\n                # renamed the attribute param to parameter, as this is a required\n                # OpenML convention - this also guards against name collisions\n                # with the required trace attributes\n                attribute = (PREFIX + key[6:], type)  # type: ignore\n                trace_attributes.append(attribute)\n\n        return OpenMLRunTrace.generate(\n            trace_attributes,\n            trace_content,\n        )\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.can_handle_flow","title":"<code>can_handle_flow(flow)</code>  <code>classmethod</code>","text":"<p>Check whether a given describes a scikit-learn estimator.</p> <p>This is done by parsing the <code>external_version</code> field.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_flow(cls, flow: OpenMLFlow) -&gt; bool:\n    \"\"\"Check whether a given describes a scikit-learn estimator.\n\n    This is done by parsing the ``external_version`` field.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return cls._is_sklearn_flow(flow)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.can_handle_model","title":"<code>can_handle_model(model)</code>  <code>classmethod</code>","text":"<p>Check whether a model is an instance of <code>sklearn.base.BaseEstimator</code>.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef can_handle_model(cls, model: Any) -&gt; bool:\n    \"\"\"Check whether a model is an instance of ``sklearn.base.BaseEstimator``.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    return isinstance(model, sklearn.base.BaseEstimator)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.check_if_model_fitted","title":"<code>check_if_model_fitted(model)</code>","text":"<p>Returns True/False denoting if the model has already been fitted/trained</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def check_if_model_fitted(self, model: Any) -&gt; bool:\n    \"\"\"Returns True/False denoting if the model has already been fitted/trained\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    from sklearn.exceptions import NotFittedError\n    from sklearn.utils.validation import check_is_fitted\n\n    try:\n        # check if model is fitted\n        check_is_fitted(model)\n\n        # Creating random dummy data of arbitrary size\n        dummy_data = np.random.uniform(size=(10, 3))  # noqa: NPY002\n        # Using 'predict' instead of 'sklearn.utils.validation.check_is_fitted' for a more\n        # robust check that works across sklearn versions and models. Internally, 'predict'\n        # should call 'check_is_fitted' for every concerned attribute, thus offering a more\n        # assured check than explicit calls to 'check_is_fitted'\n        model.predict(dummy_data)\n        # Will reach here if the model was fit on a dataset with 3 features\n        return True\n    except NotFittedError:  # needs to be the first exception to be caught\n        # Model is not fitted, as is required\n        return False\n    except ValueError:\n        # Will reach here if the model was fit on a dataset with more or less than 3 features\n        return True\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.create_setup_string","title":"<code>create_setup_string(model)</code>","text":"<p>Create a string which can be used to reinstantiate the given model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def create_setup_string(self, model: Any) -&gt; str:  # noqa: ARG002\n    \"\"\"Create a string which can be used to reinstantiate the given model.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    str\n    \"\"\"\n    return \" \".join(self.get_version_information())\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.flow_to_model","title":"<code>flow_to_model(flow, initialize_with_defaults=False, strict_version=True)</code>","text":"<p>Initializes a sklearn model based on a flow.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>mixed</code> <p>the object to deserialize (can be flow object, or any serialized parameter value that is accepted by)</p> required <code>initialize_with_defaults</code> <code>(bool, optional(default=False))</code> <p>If this flag is set, the hyperparameter values of flows will be ignored and a flow with its defaults is returned.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Type Description <code>mixed</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def flow_to_model(\n    self,\n    flow: OpenMLFlow,\n    initialize_with_defaults: bool = False,  # noqa: FBT001, FBT002\n    strict_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; Any:\n    \"\"\"Initializes a sklearn model based on a flow.\n\n    Parameters\n    ----------\n    flow : mixed\n        the object to deserialize (can be flow object, or any serialized\n        parameter value that is accepted by)\n\n    initialize_with_defaults : bool, optional (default=False)\n        If this flag is set, the hyperparameter values of flows will be\n        ignored and a flow with its defaults is returned.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    mixed\n    \"\"\"\n    return self._deserialize_sklearn(\n        flow,\n        initialize_with_defaults=initialize_with_defaults,\n        strict_version=strict_version,\n    )\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.get_version_information","title":"<code>get_version_information()</code>","text":"<p>List versions of libraries required by the flow.</p> <p>Libraries listed are <code>Python</code>, <code>scikit-learn</code>, <code>numpy</code> and <code>scipy</code>.</p> <p>Returns:</p> Type Description <code>List</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def get_version_information(self) -&gt; list[str]:\n    \"\"\"List versions of libraries required by the flow.\n\n    Libraries listed are ``Python``, ``scikit-learn``, ``numpy`` and ``scipy``.\n\n    Returns\n    -------\n    List\n    \"\"\"\n    # This can possibly be done by a package such as pyxb, but I could not get\n    # it to work properly.\n    import numpy\n    import scipy\n    import sklearn\n\n    major, minor, micro, _, _ = sys.version_info\n    python_version = \"Python_{}.\".format(\".\".join([str(major), str(minor), str(micro)]))\n    sklearn_version = f\"Sklearn_{sklearn.__version__}.\"\n    numpy_version = f\"NumPy_{numpy.__version__}.\"  # type: ignore\n    scipy_version = f\"SciPy_{scipy.__version__}.\"\n\n    return [python_version, sklearn_version, numpy_version, scipy_version]\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.instantiate_model_from_hpo_class","title":"<code>instantiate_model_from_hpo_class(model, trace_iteration)</code>","text":"<p>Instantiate a <code>base_estimator</code> which can be searched over by the hyperparameter optimization model.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> <p>A hyperparameter optimization model which defines the model to be instantiated.</p> required <code>trace_iteration</code> <code>OpenMLTraceIteration</code> <p>Describing the hyperparameter settings to instantiate.</p> required <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def instantiate_model_from_hpo_class(\n    self,\n    model: Any,\n    trace_iteration: OpenMLTraceIteration,\n) -&gt; Any:\n    \"\"\"Instantiate a ``base_estimator`` which can be searched over by the hyperparameter\n    optimization model.\n\n    Parameters\n    ----------\n    model : Any\n        A hyperparameter optimization model which defines the model to be instantiated.\n    trace_iteration : OpenMLTraceIteration\n        Describing the hyperparameter settings to instantiate.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n    if not self._is_hpo_class(model):\n        raise AssertionError(\n            \"Flow model %s is not an instance of sklearn.model_selection._search.BaseSearchCV\"\n            % model,\n        )\n    base_estimator = model.estimator\n    base_estimator.set_params(**trace_iteration.get_parameters())\n    return base_estimator\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.is_estimator","title":"<code>is_estimator(model)</code>","text":"<p>Check whether the given model is a scikit-learn estimator.</p> <p>This function is only required for backwards compatibility and will be removed in the near future.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>bool</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def is_estimator(self, model: Any) -&gt; bool:\n    \"\"\"Check whether the given model is a scikit-learn estimator.\n\n    This function is only required for backwards compatibility and will be removed in the\n    near future.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    bool\n    \"\"\"\n    o = model\n    return hasattr(o, \"fit\") and hasattr(o, \"get_params\") and hasattr(o, \"set_params\")\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.model_to_flow","title":"<code>model_to_flow(model)</code>","text":"<p>Transform a scikit-learn model to a flow for uploading it to OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Any</code> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def model_to_flow(self, model: Any) -&gt; OpenMLFlow:\n    \"\"\"Transform a scikit-learn model to a flow for uploading it to OpenML.\n\n    Parameters\n    ----------\n    model : Any\n\n    Returns\n    -------\n    OpenMLFlow\n    \"\"\"\n    # Necessary to make pypy not complain about all the different possible return types\n    return self._serialize_sklearn(model)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.obtain_parameter_values","title":"<code>obtain_parameter_values(flow, model=None)</code>","text":"<p>Extracts all parameter settings required for the flow from the model.</p> <p>If no explicit model is provided, the parameters will be extracted from <code>flow.model</code> instead.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)</p> required <code>model</code> <code>Any</code> <p>The model from which to obtain the parameter values. Must match the flow signature. If None, use the model specified in <code>OpenMLFlow.model</code>.</p> <code>None</code> <p>Returns:</p> Type Description <code>list</code> <p>A list of dicts, where each dict has the following entries: - <code>oml:name</code> : str: The OpenML parameter name - <code>oml:value</code> : mixed: A representation of the parameter value - <code>oml:component</code> : int: flow id to which the parameter belongs</p> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def obtain_parameter_values(  # noqa: C901, PLR0915\n    self,\n    flow: OpenMLFlow,\n    model: Any = None,\n) -&gt; list[dict[str, Any]]:\n    \"\"\"Extracts all parameter settings required for the flow from the model.\n\n    If no explicit model is provided, the parameters will be extracted from `flow.model`\n    instead.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        OpenMLFlow object (containing flow ids, i.e., it has to be downloaded from the server)\n\n    model: Any, optional (default=None)\n        The model from which to obtain the parameter values. Must match the flow signature.\n        If None, use the model specified in ``OpenMLFlow.model``.\n\n    Returns\n    -------\n    list\n        A list of dicts, where each dict has the following entries:\n        - ``oml:name`` : str: The OpenML parameter name\n        - ``oml:value`` : mixed: A representation of the parameter value\n        - ``oml:component`` : int: flow id to which the parameter belongs\n    \"\"\"\n    openml.flows.functions._check_flow_for_server_id(flow)\n\n    def get_flow_dict(_flow):\n        flow_map = {_flow.name: _flow.flow_id}\n        for subflow in _flow.components:\n            flow_map.update(get_flow_dict(_flow.components[subflow]))\n        return flow_map\n\n    def extract_parameters(  # noqa: PLR0915, PLR0912, C901\n        _flow,\n        _flow_dict,\n        component_model,\n        _main_call=False,  # noqa: FBT002\n        main_id=None,\n    ):\n        def is_subcomponent_specification(values):\n            # checks whether the current value can be a specification of\n            # subcomponents, as for example the value for steps parameter\n            # (in Pipeline) or transformers parameter (in\n            # ColumnTransformer).\n            return (\n                # Specification requires list/tuple of list/tuple with\n                # at least length 2.\n                isinstance(values, (tuple, list))\n                and all(isinstance(item, (tuple, list)) and len(item) &gt; 1 for item in values)\n                # And each component needs to be a flow or interpretable string\n                and all(\n                    isinstance(item[1], openml.flows.OpenMLFlow)\n                    or (\n                        isinstance(item[1], str)\n                        and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS\n                    )\n                    for item in values\n                )\n            )\n\n        # _flow is openml flow object, _param dict maps from flow name to flow\n        # id for the main call, the param dict can be overridden (useful for\n        # unit tests / sentinels) this way, for flows without subflows we do\n        # not have to rely on _flow_dict\n        exp_parameters = set(_flow.parameters)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_parameters = set()\n        else:\n            model_parameters = set(component_model.get_params(deep=False))\n        if len(exp_parameters.symmetric_difference(model_parameters)) != 0:\n            flow_params = sorted(exp_parameters)\n            model_params = sorted(model_parameters)\n            raise ValueError(\n                \"Parameters of the model do not match the \"\n                \"parameters expected by the \"\n                \"flow:\\nexpected flow parameters: \"\n                f\"{flow_params}\\nmodel parameters: {model_params}\",\n            )\n        exp_components = set(_flow.components)\n        if (\n            isinstance(component_model, str)\n            and component_model in SKLEARN_PIPELINE_STRING_COMPONENTS\n        ):\n            model_components = set()\n        else:\n            _ = set(component_model.get_params(deep=False))\n            model_components = {\n                mp\n                for mp in component_model.get_params(deep=True)\n                if \"__\" not in mp and mp not in _\n            }\n        if len(exp_components.symmetric_difference(model_components)) != 0:\n            is_problem = True\n            if len(exp_components - model_components) &gt; 0:\n                # If an expected component is not returned as a component by get_params(),\n                # this means that it is also a parameter -&gt; we need to check that this is\n                # actually the case\n                difference = exp_components - model_components\n                component_in_model_parameters = []\n                for component in difference:\n                    if component in model_parameters:\n                        component_in_model_parameters.append(True)\n                    else:\n                        component_in_model_parameters.append(False)\n                is_problem = not all(component_in_model_parameters)\n            if is_problem:\n                flow_components = sorted(exp_components)\n                model_components = sorted(model_components)\n                raise ValueError(\n                    \"Subcomponents of the model do not match the \"\n                    \"parameters expected by the \"\n                    \"flow:\\nexpected flow subcomponents: \"\n                    f\"{flow_components}\\nmodel subcomponents: {model_components}\",\n                )\n\n        _params = []\n        for _param_name in _flow.parameters:\n            _current = OrderedDict()\n            _current[\"oml:name\"] = _param_name\n\n            current_param_values = self.model_to_flow(component_model.get_params()[_param_name])\n\n            # Try to filter out components (a.k.a. subflows) which are\n            # handled further down in the code (by recursively calling\n            # this function)!\n            if isinstance(current_param_values, openml.flows.OpenMLFlow):\n                continue\n\n            if is_subcomponent_specification(current_param_values):\n                # complex parameter value, with subcomponents\n                parsed_values = []\n                for subcomponent in current_param_values:\n                    # scikit-learn stores usually tuples in the form\n                    # (name (str), subcomponent (mixed), argument\n                    # (mixed)). OpenML replaces the subcomponent by an\n                    # OpenMLFlow object.\n                    if len(subcomponent) &lt; 2 or len(subcomponent) &gt; 3:\n                        raise ValueError(\"Component reference should be \" \"size {2,3}. \")\n\n                    subcomponent_identifier = subcomponent[0]\n                    subcomponent_flow = subcomponent[1]\n                    if not isinstance(subcomponent_identifier, str):\n                        raise TypeError(\n                            \"Subcomponent identifier should be of type string, \"\n                            f\"but is {type(subcomponent_identifier)}\",\n                        )\n                    if not isinstance(subcomponent_flow, (openml.flows.OpenMLFlow, str)):\n                        if (\n                            isinstance(subcomponent_flow, str)\n                            and subcomponent_flow in SKLEARN_PIPELINE_STRING_COMPONENTS\n                        ):\n                            pass\n                        else:\n                            raise TypeError(\n                                \"Subcomponent flow should be of type flow, but is {}\".format(\n                                    type(subcomponent_flow),\n                                ),\n                            )\n\n                    current = {\n                        \"oml-python:serialized_object\": COMPONENT_REFERENCE,\n                        \"value\": {\n                            \"key\": subcomponent_identifier,\n                            \"step_name\": subcomponent_identifier,\n                        },\n                    }\n                    if len(subcomponent) == 3:\n                        if not isinstance(subcomponent[2], list) and not isinstance(\n                            subcomponent[2],\n                            OrderedDict,\n                        ):\n                            raise TypeError(\n                                \"Subcomponent argument should be list or OrderedDict\",\n                            )\n                        current[\"value\"][\"argument_1\"] = subcomponent[2]\n                    parsed_values.append(current)\n                parsed_values = json.dumps(parsed_values)\n            else:\n                # vanilla parameter value\n                parsed_values = json.dumps(current_param_values)\n\n            _current[\"oml:value\"] = parsed_values\n            if _main_call:\n                _current[\"oml:component\"] = main_id\n            else:\n                _current[\"oml:component\"] = _flow_dict[_flow.name]\n            _params.append(_current)\n\n        for _identifier in _flow.components:\n            subcomponent_model = component_model.get_params()[_identifier]\n            _params.extend(\n                extract_parameters(\n                    _flow.components[_identifier],\n                    _flow_dict,\n                    subcomponent_model,\n                ),\n            )\n        return _params\n\n    flow_dict = get_flow_dict(flow)\n    model = model if model is not None else flow.model\n    return extract_parameters(flow, flow_dict, model, _main_call=True, main_id=flow.flow_id)\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.seed_model","title":"<code>seed_model(model, seed=None)</code>","text":"<p>Set the random state of all the unseeded components of a model and return the seeded model.</p> <p>Required so that all seed information can be uploaded to OpenML for reproducible results.</p> <p>Models that are already seeded will maintain the seed. In this case, only integer seeds are allowed (An exception is raised when a RandomState was used as seed).</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>The model to be seeded</p> required <code>seed</code> <code>int</code> <p>The seed to initialize the RandomState with. Unseeded subcomponents will be seeded with a random number from the RandomState.</p> <code>None</code> <p>Returns:</p> Type Description <code>Any</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>def seed_model(self, model: Any, seed: int | None = None) -&gt; Any:  # noqa: C901\n    \"\"\"Set the random state of all the unseeded components of a model and return the seeded\n    model.\n\n    Required so that all seed information can be uploaded to OpenML for reproducible results.\n\n    Models that are already seeded will maintain the seed. In this case,\n    only integer seeds are allowed (An exception is raised when a RandomState was used as\n    seed).\n\n    Parameters\n    ----------\n    model : sklearn model\n        The model to be seeded\n    seed : int\n        The seed to initialize the RandomState with. Unseeded subcomponents\n        will be seeded with a random number from the RandomState.\n\n    Returns\n    -------\n    Any\n    \"\"\"\n\n    def _seed_current_object(current_value):\n        if isinstance(current_value, int):  # acceptable behaviour\n            return False\n\n        if isinstance(current_value, np.random.RandomState):\n            raise ValueError(\n                \"Models initialized with a RandomState object are not \"\n                \"supported. Please seed with an integer. \",\n            )\n\n        if current_value is not None:\n            raise ValueError(\n                \"Models should be seeded with int or None (this should never \" \"happen). \",\n            )\n\n        return True\n\n    rs = np.random.RandomState(seed)\n    model_params = model.get_params()\n    random_states = {}\n    for param_name in sorted(model_params):\n        if \"random_state\" in param_name:\n            current_value = model_params[param_name]\n            # important to draw the value at this point (and not in the if\n            # statement) this way we guarantee that if a different set of\n            # subflows is seeded, the same number of the random generator is\n            # used\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                random_states[param_name] = new_value\n\n        # Also seed CV objects!\n        elif isinstance(model_params[param_name], sklearn.model_selection.BaseCrossValidator):\n            if not hasattr(model_params[param_name], \"random_state\"):\n                continue\n\n            current_value = model_params[param_name].random_state\n            new_value = rs.randint(0, 2**16)\n            if _seed_current_object(current_value):\n                model_params[param_name].random_state = new_value\n\n    model.set_params(**random_states)\n    return model\n</code></pre>"},{"location":"reference/extensions/sklearn/extension/#openml.extensions.sklearn.extension.SklearnExtension.trim_flow_name","title":"<code>trim_flow_name(long_name, extra_trim_length=100, _outer=True)</code>  <code>classmethod</code>","text":"<p>Shorten generated sklearn flow name to at most <code>max_length</code> characters.</p> <p>Flows are assumed to have the following naming structure: <code>(model_selection)? (pipeline)? (steps)+</code> and will be shortened to: <code>sklearn.(selection.)?(pipeline.)?(steps)+</code> e.g. (white spaces and newlines added for readability)</p> <p>.. code ::</p> <pre><code>sklearn.pipeline.Pipeline(\n    columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n        numeric=sklearn.pipeline.Pipeline(\n            imputer=sklearn.preprocessing.imputation.Imputer,\n            standardscaler=sklearn.preprocessing.data.StandardScaler),\n        nominal=sklearn.pipeline.Pipeline(\n            simpleimputer=sklearn.impute.SimpleImputer,\n            onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n    variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n    svc=sklearn.svm.classes.SVC)\n</code></pre> <p>-&gt; <code>sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)</code></p> <p>Parameters:</p> Name Type Description Default <code>long_name</code> <code>str</code> <p>The full flow name generated by the scikit-learn extension.</p> required <code>extra_trim_length</code> <code>int</code> <p>If the trimmed name would exceed <code>extra_trim_length</code> characters, additional trimming of the short name is performed. This reduces the produced short name length. There is no guarantee the end result will not exceed <code>extra_trim_length</code>.</p> <code>100</code> <code>_outer</code> <code>bool(default=True)</code> <p>For internal use only. Specifies if the function is called recursively.</p> <code>True</code> <p>Returns:</p> Type Description <code>str</code> Source code in <code>openml/extensions/sklearn/extension.py</code> <pre><code>@classmethod\ndef trim_flow_name(  # noqa: C901\n    cls,\n    long_name: str,\n    extra_trim_length: int = 100,\n    _outer: bool = True,  # noqa: FBT001, FBT002\n) -&gt; str:\n    \"\"\"Shorten generated sklearn flow name to at most ``max_length`` characters.\n\n    Flows are assumed to have the following naming structure:\n    ``(model_selection)? (pipeline)? (steps)+``\n    and will be shortened to:\n    ``sklearn.(selection.)?(pipeline.)?(steps)+``\n    e.g. (white spaces and newlines added for readability)\n\n    .. code ::\n\n        sklearn.pipeline.Pipeline(\n            columntransformer=sklearn.compose._column_transformer.ColumnTransformer(\n                numeric=sklearn.pipeline.Pipeline(\n                    imputer=sklearn.preprocessing.imputation.Imputer,\n                    standardscaler=sklearn.preprocessing.data.StandardScaler),\n                nominal=sklearn.pipeline.Pipeline(\n                    simpleimputer=sklearn.impute.SimpleImputer,\n                    onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder)),\n            variancethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,\n            svc=sklearn.svm.classes.SVC)\n\n    -&gt;\n    ``sklearn.Pipeline(ColumnTransformer,VarianceThreshold,SVC)``\n\n    Parameters\n    ----------\n    long_name : str\n        The full flow name generated by the scikit-learn extension.\n    extra_trim_length: int (default=100)\n        If the trimmed name would exceed `extra_trim_length` characters, additional trimming\n        of the short name is performed. This reduces the produced short name length.\n        There is no guarantee the end result will not exceed `extra_trim_length`.\n    _outer : bool (default=True)\n        For internal use only. Specifies if the function is called recursively.\n\n    Returns\n    -------\n    str\n\n    \"\"\"\n\n    def remove_all_in_parentheses(string: str) -&gt; str:\n        string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        while removals &gt; 0:\n            string, removals = re.subn(r\"\\([^()]*\\)\", \"\", string)\n        return string\n\n    # Generally, we want to trim all hyperparameters, the exception to that is for model\n    # selection, as the `estimator` hyperparameter is very indicative of what is in the flow.\n    # So we first trim name of the `estimator` specified in mode selection. For reference, in\n    # the example below, we want to trim `sklearn.tree.tree.DecisionTreeClassifier`, and\n    # keep it in the final trimmed flow name:\n    # sklearn.pipeline.Pipeline(Imputer=sklearn.preprocessing.imputation.Imputer,\n    # VarianceThreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,  # noqa: ERA001, E501\n    # Estimator=sklearn.model_selection._search.RandomizedSearchCV(estimator=\n    # sklearn.tree.tree.DecisionTreeClassifier))\n    if \"sklearn.model_selection\" in long_name:\n        start_index = long_name.index(\"sklearn.model_selection\")\n        estimator_start = (\n            start_index + long_name[start_index:].index(\"estimator=\") + len(\"estimator=\")\n        )\n\n        model_select_boilerplate = long_name[start_index:estimator_start]\n        # above is .g. \"sklearn.model_selection._search.RandomizedSearchCV(estimator=\"\n        model_selection_class = model_select_boilerplate.split(\"(\")[0].split(\".\")[-1]\n\n        # Now we want to also find and parse the `estimator`, for this we find the closing\n        # parenthesis to the model selection technique:\n        closing_parenthesis_expected = 1\n        for char in long_name[estimator_start:]:\n            if char == \"(\":\n                closing_parenthesis_expected += 1\n            if char == \")\":\n                closing_parenthesis_expected -= 1\n            if closing_parenthesis_expected == 0:\n                break\n\n        _end: int = estimator_start + len(long_name[estimator_start:]) - 1\n        model_select_pipeline = long_name[estimator_start:_end]\n\n        trimmed_pipeline = cls.trim_flow_name(model_select_pipeline, _outer=False)\n        _, trimmed_pipeline = trimmed_pipeline.split(\".\", maxsplit=1)  # trim module prefix\n        model_select_short = f\"sklearn.{model_selection_class}[{trimmed_pipeline}]\"\n        name = long_name[:start_index] + model_select_short + long_name[_end + 1 :]\n    else:\n        name = long_name\n\n    module_name = long_name.split(\".\")[0]\n    short_name = module_name + \".{}\"\n\n    if name.startswith(\"sklearn.pipeline\"):\n        full_pipeline_class, pipeline = name[:-1].split(\"(\", maxsplit=1)\n        pipeline_class = full_pipeline_class.split(\".\")[-1]\n        # We don't want nested pipelines in the short name, so we trim all complicated\n        # subcomponents, i.e. those with parentheses:\n        pipeline = remove_all_in_parentheses(pipeline)\n\n        # then the pipeline steps are formatted e.g.:\n        # step1name=sklearn.submodule.ClassName,step2name...\n        components = [component.split(\".\")[-1] for component in pipeline.split(\",\")]\n        pipeline = \"{}({})\".format(pipeline_class, \",\".join(components))\n        if len(short_name.format(pipeline)) &gt; extra_trim_length:\n            pipeline = f\"{pipeline_class}(...,{components[-1]})\"\n    else:\n        # Just a simple component: e.g. sklearn.tree.DecisionTreeClassifier\n        pipeline = remove_all_in_parentheses(name).split(\".\")[-1]\n\n    if not _outer:\n        # Anything from parenthesis in inner calls should not be culled, so we use brackets\n        pipeline = pipeline.replace(\"(\", \"[\").replace(\")\", \"]\")\n    else:\n        # Square brackets may be introduced with nested model_selection\n        pipeline = pipeline.replace(\"[\", \"(\").replace(\"]\", \")\")\n\n    return short_name.format(pipeline)\n</code></pre>"},{"location":"reference/flows/","title":"flows","text":""},{"location":"reference/flows/#openml.flows.OpenMLFlow","title":"<code>OpenMLFlow</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Flow. Stores machine learning models.</p> <p>Flows should not be generated manually, but by the function :meth:<code>openml.flows.create_flow_from_model</code>. Using this helper function ensures that all relevant fields are filled in.</p> <p>Implements <code>openml.implementation.upload.xsd &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/ openml.implementation.upload.xsd&gt;</code>_.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the flow. Is used together with the attribute <code>external_version</code> as a unique identifier of the flow.</p> required <code>description</code> <code>str</code> <p>Human-readable description of the flow (free text).</p> required <code>model</code> <code>object</code> <p>ML model which is described by this flow.</p> required <code>components</code> <code>OrderedDict</code> <p>Mapping from component identifier to an OpenMLFlow object. Components are usually subfunctions of an algorithm (e.g. kernels), base learners in ensemble algorithms (decision tree in adaboost) or building blocks of a machine learning pipeline. Components are modeled as independent flows and can be shared between flows (different pipelines can use the same components).</p> required <code>parameters</code> <code>OrderedDict</code> <p>Mapping from parameter name to the parameter default value. The parameter default value must be of type <code>str</code>, so that the respective toolbox plugin can take care of casting the parameter default value to the correct type.</p> required <code>parameters_meta_info</code> <code>OrderedDict</code> <p>Mapping from parameter name to <code>dict</code>. Stores additional information for each parameter. Required keys are <code>data_type</code> and <code>description</code>.</p> required <code>external_version</code> <code>str</code> <p>Version number of the software the flow is implemented in. Is used together with the attribute <code>name</code> as a uniquer identifier of the flow.</p> required <code>tags</code> <code>list</code> <p>List of tags. Created on the server by other API calls.</p> required <code>language</code> <code>str</code> <p>Natural language the flow is described in (not the programming language).</p> required <code>dependencies</code> <code>str</code> <p>A list of dependencies necessary to run the flow. This field should contain all libraries the flow depends on. To allow reproducibility it should also specify the exact version numbers.</p> required <code>class_name</code> <code>str</code> <p>The development language name of the class which is described by this flow.</p> <code>None</code> <code>custom_name</code> <code>str</code> <p>Custom name of the flow given by the owner.</p> <code>None</code> <code>binary_url</code> <code>str</code> <p>Url from which the binary can be downloaded. Added by the server. Ignored when uploaded manually. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_format</code> <code>str</code> <p>Format in which the binary code was uploaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_md5</code> <code>str</code> <p>MD5 checksum to check if the binary code was correctly downloaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>uploader</code> <code>str</code> <p>OpenML user ID of the uploader. Filled in by the server.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>Date the flow was uploaded. Filled in by the server.</p> <code>None</code> <code>flow_id</code> <code>int</code> <p>Flow ID. Assigned by the server.</p> <code>None</code> <code>extension</code> <code>Extension</code> <p>The extension for a flow (e.g., sklearn).</p> <code>None</code> <code>version</code> <code>str</code> <p>OpenML version of the flow. Assigned by the server.</p> <code>None</code> Source code in <code>openml/flows/flow.py</code> <pre><code>class OpenMLFlow(OpenMLBase):\n    \"\"\"OpenML Flow. Stores machine learning models.\n\n    Flows should not be generated manually, but by the function\n    :meth:`openml.flows.create_flow_from_model`. Using this helper function\n    ensures that all relevant fields are filled in.\n\n    Implements `openml.implementation.upload.xsd\n    &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/\n    openml.implementation.upload.xsd&gt;`_.\n\n    Parameters\n    ----------\n    name : str\n        Name of the flow. Is used together with the attribute\n        `external_version` as a unique identifier of the flow.\n    description : str\n        Human-readable description of the flow (free text).\n    model : object\n        ML model which is described by this flow.\n    components : OrderedDict\n        Mapping from component identifier to an OpenMLFlow object. Components\n        are usually subfunctions of an algorithm (e.g. kernels), base learners\n        in ensemble algorithms (decision tree in adaboost) or building blocks\n        of a machine learning pipeline. Components are modeled as independent\n        flows and can be shared between flows (different pipelines can use\n        the same components).\n    parameters : OrderedDict\n        Mapping from parameter name to the parameter default value. The\n        parameter default value must be of type `str`, so that the respective\n        toolbox plugin can take care of casting the parameter default value to\n        the correct type.\n    parameters_meta_info : OrderedDict\n        Mapping from parameter name to `dict`. Stores additional information\n        for each parameter. Required keys are `data_type` and `description`.\n    external_version : str\n        Version number of the software the flow is implemented in. Is used\n        together with the attribute `name` as a uniquer identifier of the flow.\n    tags : list\n        List of tags. Created on the server by other API calls.\n    language : str\n        Natural language the flow is described in (not the programming\n        language).\n    dependencies : str\n        A list of dependencies necessary to run the flow. This field should\n        contain all libraries the flow depends on. To allow reproducibility\n        it should also specify the exact version numbers.\n    class_name : str, optional\n        The development language name of the class which is described by this\n        flow.\n    custom_name : str, optional\n        Custom name of the flow given by the owner.\n    binary_url : str, optional\n        Url from which the binary can be downloaded. Added by the server.\n        Ignored when uploaded manually. Will not be used by the python API\n        because binaries aren't compatible across machines.\n    binary_format : str, optional\n        Format in which the binary code was uploaded. Will not be used by the\n        python API because binaries aren't compatible across machines.\n    binary_md5 : str, optional\n        MD5 checksum to check if the binary code was correctly downloaded. Will\n        not be used by the python API because binaries aren't compatible across\n        machines.\n    uploader : str, optional\n        OpenML user ID of the uploader. Filled in by the server.\n    upload_date : str, optional\n        Date the flow was uploaded. Filled in by the server.\n    flow_id : int, optional\n        Flow ID. Assigned by the server.\n    extension : Extension, optional\n        The extension for a flow (e.g., sklearn).\n    version : str, optional\n        OpenML version of the flow. Assigned by the server.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        name: str,\n        description: str,\n        model: object,\n        components: dict,\n        parameters: dict,\n        parameters_meta_info: dict,\n        external_version: str,\n        tags: list,\n        language: str,\n        dependencies: str,\n        class_name: str | None = None,\n        custom_name: str | None = None,\n        binary_url: str | None = None,\n        binary_format: str | None = None,\n        binary_md5: str | None = None,\n        uploader: str | None = None,\n        upload_date: str | None = None,\n        flow_id: int | None = None,\n        extension: Extension | None = None,\n        version: str | None = None,\n    ):\n        self.name = name\n        self.description = description\n        self.model = model\n\n        for variable, variable_name in [\n            [components, \"components\"],\n            [parameters, \"parameters\"],\n            [parameters_meta_info, \"parameters_meta_info\"],\n        ]:\n            if not isinstance(variable, (OrderedDict, dict)):\n                raise TypeError(\n                    f\"{variable_name} must be of type OrderedDict or dict, \"\n                    f\"but is {type(variable)}.\",\n                )\n\n        self.components = components\n        self.parameters = parameters\n        self.parameters_meta_info = parameters_meta_info\n        self.class_name = class_name\n\n        keys_parameters = set(parameters.keys())\n        keys_parameters_meta_info = set(parameters_meta_info.keys())\n        if len(keys_parameters.difference(keys_parameters_meta_info)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters, but not in \"\n                \"parameters_meta_info.\"\n                % str(keys_parameters.difference(keys_parameters_meta_info)),\n            )\n        if len(keys_parameters_meta_info.difference(keys_parameters)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters_meta_info, \"\n                \"but not in parameters.\"\n                % str(keys_parameters_meta_info.difference(keys_parameters)),\n            )\n\n        self.external_version = external_version\n        self.uploader = uploader\n\n        self.custom_name = custom_name\n        self.tags = tags if tags is not None else []\n        self.binary_url = binary_url\n        self.binary_format = binary_format\n        self.binary_md5 = binary_md5\n        self.version = version\n        self.upload_date = upload_date\n        self.language = language\n        self.dependencies = dependencies\n        self.flow_id = flow_id\n        if extension is None:\n            self._extension = get_extension_by_flow(self)\n        else:\n            self._extension = extension\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the flow.\"\"\"\n        return self.flow_id\n\n    @property\n    def extension(self) -&gt; Extension:\n        \"\"\"The extension of the flow (e.g., sklearn).\"\"\"\n        if self._extension is not None:\n            return self._extension\n\n        raise RuntimeError(\n            f\"No extension could be found for flow {self.flow_id}: {self.name}\",\n        )\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields = {\n            \"Flow Name\": self.name,\n            \"Flow Description\": self.description,\n            \"Dependencies\": self.dependencies,\n        }\n        if self.flow_id is not None:\n            fields[\"Flow URL\"] = self.openml_url if self.openml_url is not None else \"None\"\n            fields[\"Flow ID\"] = str(self.flow_id)\n            if self.version is not None:\n                fields[\"Flow ID\"] += f\" (version {self.version})\"\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.binary_url is not None:\n            fields[\"Binary URL\"] = self.binary_url\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Flow ID\",\n            \"Flow URL\",\n            \"Flow Name\",\n            \"Flow Description\",\n            \"Binary URL\",\n            \"Upload Date\",\n            \"Dependencies\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: C901, PLR0912\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        flow_container = OrderedDict()  # type: 'dict[str, dict]'\n        flow_dict = OrderedDict(\n            [(\"@xmlns:oml\", \"http://openml.org/openml\")],\n        )  # type: 'dict[str, list | str]'  # E501\n        flow_container[\"oml:flow\"] = flow_dict\n        _add_if_nonempty(flow_dict, \"oml:id\", self.flow_id)\n\n        for required in [\"name\", \"external_version\"]:\n            if getattr(self, required) is None:\n                raise ValueError(f\"self.{required} is required but None\")\n        for attribute in [\n            \"uploader\",\n            \"name\",\n            \"custom_name\",\n            \"class_name\",\n            \"version\",\n            \"external_version\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n        ]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        if not self.description:\n            logger = logging.getLogger(__name__)\n            logger.warning(\"Flow % has empty description\", self.name)\n\n        flow_parameters = []\n        for key in self.parameters:\n            param_dict = OrderedDict()  # type: 'OrderedDict[str, str]'\n            param_dict[\"oml:name\"] = key\n            meta_info = self.parameters_meta_info[key]\n\n            _add_if_nonempty(param_dict, \"oml:data_type\", meta_info[\"data_type\"])\n            param_dict[\"oml:default_value\"] = self.parameters[key]\n            _add_if_nonempty(param_dict, \"oml:description\", meta_info[\"description\"])\n\n            for key_, value in param_dict.items():\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n                if value is not None and not isinstance(value, str):\n                    raise ValueError(\n                        f\"Parameter value {value} cannot be serialized \"\n                        f\"because it is of type {type(value)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            flow_parameters.append(param_dict)\n\n        flow_dict[\"oml:parameter\"] = flow_parameters\n\n        components = []\n        for key in self.components:\n            component_dict = OrderedDict()  # type: 'OrderedDict[str, dict]'\n            component_dict[\"oml:identifier\"] = key\n            if self.components[key] in [\"passthrough\", \"drop\"]:\n                component_dict[\"oml:flow\"] = {\n                    \"oml-python:serialized_object\": \"component_reference\",\n                    \"value\": {\"key\": self.components[key], \"step_name\": self.components[key]},\n                }\n            else:\n                component_dict[\"oml:flow\"] = self.components[key]._to_dict()[\"oml:flow\"]\n\n            for key_ in component_dict:\n                # We only need to check if the key is a string, because the\n                # value is a flow. The flow itself is valid by recursion\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            components.append(component_dict)\n\n        flow_dict[\"oml:component\"] = components\n        flow_dict[\"oml:tag\"] = self.tags\n        for attribute in [\"binary_url\", \"binary_format\", \"binary_md5\"]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        return flow_container\n\n    @classmethod\n    def _from_dict(cls, xml_dict: dict) -&gt; OpenMLFlow:\n        \"\"\"Create a flow from an xml description.\n\n        Calls itself recursively to create :class:`OpenMLFlow` objects of\n        subflows (components).\n\n        XML definition of a flow is available at\n        https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd\n\n        Parameters\n        ----------\n        xml_dict : dict\n            Dictionary representation of the flow as created by _to_dict()\n\n        Returns\n        -------\n            OpenMLFlow\n\n        \"\"\"  # E501\n        arguments = OrderedDict()\n        dic = xml_dict[\"oml:flow\"]\n\n        # Mandatory parts in the xml file\n        for key in [\"name\"]:\n            arguments[key] = dic[\"oml:\" + key]\n\n        # non-mandatory parts in the xml file\n        for key in [\n            \"external_version\",\n            \"uploader\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n            \"version\",\n            \"binary_url\",\n            \"binary_format\",\n            \"binary_md5\",\n            \"class_name\",\n            \"custom_name\",\n        ]:\n            arguments[key] = dic.get(\"oml:\" + key)\n\n        # has to be converted to an int if present and cannot parsed in the\n        # two loops above\n        arguments[\"flow_id\"] = int(dic[\"oml:id\"]) if dic.get(\"oml:id\") is not None else None\n\n        # Now parse parts of a flow which can occur multiple times like\n        # parameters, components (subflows) and tags. These can't be tackled\n        # in the loops above because xmltodict returns a dict if such an\n        # entity occurs once, and a list if it occurs multiple times.\n        # Furthermore, they must be treated differently, for example\n        # for components this method is called recursively and\n        # for parameters the actual information is split into two dictionaries\n        # for easier access in python.\n\n        parameters = OrderedDict()\n        parameters_meta_info = OrderedDict()\n        if \"oml:parameter\" in dic:\n            # In case of a single parameter, xmltodict returns a dictionary,\n            # otherwise a list.\n            oml_parameters = extract_xml_tags(\"oml:parameter\", dic, allow_none=False)\n\n            for oml_parameter in oml_parameters:\n                parameter_name = oml_parameter[\"oml:name\"]\n                default_value = oml_parameter[\"oml:default_value\"]\n                parameters[parameter_name] = default_value\n\n                meta_info = OrderedDict()\n                meta_info[\"description\"] = oml_parameter.get(\"oml:description\")\n                meta_info[\"data_type\"] = oml_parameter.get(\"oml:data_type\")\n                parameters_meta_info[parameter_name] = meta_info\n        arguments[\"parameters\"] = parameters\n        arguments[\"parameters_meta_info\"] = parameters_meta_info\n\n        components = OrderedDict()\n        if \"oml:component\" in dic:\n            # In case of a single component xmltodict returns a dict,\n            # otherwise a list.\n            oml_components = extract_xml_tags(\"oml:component\", dic, allow_none=False)\n\n            for component in oml_components:\n                flow = OpenMLFlow._from_dict(component)\n                components[component[\"oml:identifier\"]] = flow\n        arguments[\"components\"] = components\n        arguments[\"tags\"] = extract_xml_tags(\"oml:tag\", dic)\n\n        arguments[\"model\"] = None\n        return cls(**arguments)\n\n    def to_filesystem(self, output_directory: str | Path) -&gt; None:\n        \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n        output_directory = Path(output_directory)\n        output_directory.mkdir(parents=True, exist_ok=True)\n\n        output_path = output_directory / \"flow.xml\"\n        if output_path.exists():\n            raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n        run_xml = self._to_xml()\n        with output_path.open(\"w\") as f:\n            f.write(run_xml)\n\n    @classmethod\n    def from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n        \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n        input_directory = Path(input_directory) / \"flow.xml\"\n        with input_directory.open() as f:\n            xml_string = f.read()\n        return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.flow_id = int(xml_response[\"oml:upload_flow\"][\"oml:id\"])\n\n    def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n        \"\"\"Publish this flow to OpenML server.\n\n        Raises a PyOpenMLError if the flow exists on the server, but\n        `self.flow_id` does not match the server known flow id.\n\n        Parameters\n        ----------\n        raise_error_if_exists : bool, optional (default=False)\n            If True, raise PyOpenMLError if the flow exists on the server.\n            If False, update the local flow to match the server flow.\n\n        Returns\n        -------\n        self : OpenMLFlow\n\n        \"\"\"\n        # Import at top not possible because of cyclic dependencies. In\n        # particular, flow.py tries to import functions.py in order to call\n        # get_flow(), while functions.py tries to import flow.py in order to\n        # instantiate an OpenMLFlow.\n        import openml.flows.functions\n\n        flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n        if not flow_id:\n            if self.flow_id:\n                raise openml.exceptions.PyOpenMLError(\n                    \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n                )\n            super().publish()\n            assert self.flow_id is not None  # for mypy\n            flow_id = self.flow_id\n        elif raise_error_if_exists:\n            error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n            raise openml.exceptions.PyOpenMLError(error_message)\n        elif self.flow_id is not None and self.flow_id != flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n            )\n\n        flow = openml.flows.functions.get_flow(flow_id)\n        _copy_server_fields(flow, self)\n        try:\n            openml.flows.functions.assert_flows_equal(\n                self,\n                flow,\n                flow.upload_date,\n                ignore_parameter_values=True,\n                ignore_custom_name_if_none=True,\n            )\n        except ValueError as e:\n            message = e.args[0]\n            raise ValueError(\n                \"The flow on the server is inconsistent with the local flow. \"\n                f\"The server flow ID is {flow_id}. Please check manually and remove \"\n                f\"the flow if necessary! Error is:\\n'{message}'\",\n            ) from e\n        return self\n\n    def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n        \"\"\"\n        Returns for each sub-component of the flow the path of identifiers\n        that should be traversed to reach this component. The resulting dict\n        maps a key (identifying a flow by either its id, name or fullname) to\n        the parameter prefix.\n\n        Parameters\n        ----------\n        key_item: str\n            The flow attribute that will be used to identify flows in the\n            structure. Allowed values {flow_id, name}\n\n        Returns\n        -------\n        dict[str, List[str]]\n            The flow structure\n        \"\"\"\n        if key_item not in [\"flow_id\", \"name\"]:\n            raise ValueError(\"key_item should be in {flow_id, name}\")\n        structure = {}\n        for key, sub_flow in self.components.items():\n            sub_structure = sub_flow.get_structure(key_item)\n            for flow_name, flow_sub_structure in sub_structure.items():\n                structure[flow_name] = [key, *flow_sub_structure]\n        structure[getattr(self, key_item)] = []\n        return structure\n\n    def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n        \"\"\"\n        Returns a subflow from the tree of dependencies.\n\n        Parameters\n        ----------\n        structure: list[str]\n            A list of strings, indicating the location of the subflow\n\n        Returns\n        -------\n        OpenMLFlow\n            The OpenMLFlow that corresponds to the structure\n        \"\"\"\n        # make a copy of structure, as we don't want to change it in the\n        # outer scope\n        structure = list(structure)\n        if len(structure) &lt; 1:\n            raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n        sub_identifier = structure[0]\n        if sub_identifier not in self.components:\n            raise ValueError(\n                f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n            )\n        if len(structure) == 1:\n            return self.components[sub_identifier]  # type: ignore\n\n        structure.pop(0)\n        return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.extension","title":"<code>extension: Extension</code>  <code>property</code>","text":"<p>The extension of the flow (e.g., sklearn).</p>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the flow.</p>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.from_filesystem","title":"<code>from_filesystem(input_directory)</code>  <code>classmethod</code>","text":"<p>Read a flow from an XML in input_directory on the filesystem.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n    \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n    input_directory = Path(input_directory) / \"flow.xml\"\n    with input_directory.open() as f:\n        xml_string = f.read()\n    return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.get_structure","title":"<code>get_structure(key_item)</code>","text":"<p>Returns for each sub-component of the flow the path of identifiers that should be traversed to reach this component. The resulting dict maps a key (identifying a flow by either its id, name or fullname) to the parameter prefix.</p> <p>Parameters:</p> Name Type Description Default <code>key_item</code> <code>str</code> <p>The flow attribute that will be used to identify flows in the structure. Allowed values {flow_id, name}</p> required <p>Returns:</p> Type Description <code>dict[str, List[str]]</code> <p>The flow structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n    \"\"\"\n    Returns for each sub-component of the flow the path of identifiers\n    that should be traversed to reach this component. The resulting dict\n    maps a key (identifying a flow by either its id, name or fullname) to\n    the parameter prefix.\n\n    Parameters\n    ----------\n    key_item: str\n        The flow attribute that will be used to identify flows in the\n        structure. Allowed values {flow_id, name}\n\n    Returns\n    -------\n    dict[str, List[str]]\n        The flow structure\n    \"\"\"\n    if key_item not in [\"flow_id\", \"name\"]:\n        raise ValueError(\"key_item should be in {flow_id, name}\")\n    structure = {}\n    for key, sub_flow in self.components.items():\n        sub_structure = sub_flow.get_structure(key_item)\n        for flow_name, flow_sub_structure in sub_structure.items():\n            structure[flow_name] = [key, *flow_sub_structure]\n    structure[getattr(self, key_item)] = []\n    return structure\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.get_subflow","title":"<code>get_subflow(structure)</code>","text":"<p>Returns a subflow from the tree of dependencies.</p> <p>Parameters:</p> Name Type Description Default <code>structure</code> <code>list[str]</code> <p>A list of strings, indicating the location of the subflow</p> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> <p>The OpenMLFlow that corresponds to the structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n    \"\"\"\n    Returns a subflow from the tree of dependencies.\n\n    Parameters\n    ----------\n    structure: list[str]\n        A list of strings, indicating the location of the subflow\n\n    Returns\n    -------\n    OpenMLFlow\n        The OpenMLFlow that corresponds to the structure\n    \"\"\"\n    # make a copy of structure, as we don't want to change it in the\n    # outer scope\n    structure = list(structure)\n    if len(structure) &lt; 1:\n        raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n    sub_identifier = structure[0]\n    if sub_identifier not in self.components:\n        raise ValueError(\n            f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n        )\n    if len(structure) == 1:\n        return self.components[sub_identifier]  # type: ignore\n\n    structure.pop(0)\n    return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.publish","title":"<code>publish(raise_error_if_exists=False)</code>","text":"<p>Publish this flow to OpenML server.</p> <p>Raises a PyOpenMLError if the flow exists on the server, but <code>self.flow_id</code> does not match the server known flow id.</p> <p>Parameters:</p> Name Type Description Default <code>raise_error_if_exists</code> <code>(bool, optional(default=False))</code> <p>If True, raise PyOpenMLError if the flow exists on the server. If False, update the local flow to match the server flow.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>self</code> <code>OpenMLFlow</code> Source code in <code>openml/flows/flow.py</code> <pre><code>def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Publish this flow to OpenML server.\n\n    Raises a PyOpenMLError if the flow exists on the server, but\n    `self.flow_id` does not match the server known flow id.\n\n    Parameters\n    ----------\n    raise_error_if_exists : bool, optional (default=False)\n        If True, raise PyOpenMLError if the flow exists on the server.\n        If False, update the local flow to match the server flow.\n\n    Returns\n    -------\n    self : OpenMLFlow\n\n    \"\"\"\n    # Import at top not possible because of cyclic dependencies. In\n    # particular, flow.py tries to import functions.py in order to call\n    # get_flow(), while functions.py tries to import flow.py in order to\n    # instantiate an OpenMLFlow.\n    import openml.flows.functions\n\n    flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n    if not flow_id:\n        if self.flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n            )\n        super().publish()\n        assert self.flow_id is not None  # for mypy\n        flow_id = self.flow_id\n    elif raise_error_if_exists:\n        error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n        raise openml.exceptions.PyOpenMLError(error_message)\n    elif self.flow_id is not None and self.flow_id != flow_id:\n        raise openml.exceptions.PyOpenMLError(\n            \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n        )\n\n    flow = openml.flows.functions.get_flow(flow_id)\n    _copy_server_fields(flow, self)\n    try:\n        openml.flows.functions.assert_flows_equal(\n            self,\n            flow,\n            flow.upload_date,\n            ignore_parameter_values=True,\n            ignore_custom_name_if_none=True,\n        )\n    except ValueError as e:\n        message = e.args[0]\n        raise ValueError(\n            \"The flow on the server is inconsistent with the local flow. \"\n            f\"The server flow ID is {flow_id}. Please check manually and remove \"\n            f\"the flow if necessary! Error is:\\n'{message}'\",\n        ) from e\n    return self\n</code></pre>"},{"location":"reference/flows/#openml.flows.OpenMLFlow.to_filesystem","title":"<code>to_filesystem(output_directory)</code>","text":"<p>Write a flow to the filesystem as XML to output_directory.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def to_filesystem(self, output_directory: str | Path) -&gt; None:\n    \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n    output_directory = Path(output_directory)\n    output_directory.mkdir(parents=True, exist_ok=True)\n\n    output_path = output_directory / \"flow.xml\"\n    if output_path.exists():\n        raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n    run_xml = self._to_xml()\n    with output_path.open(\"w\") as f:\n        f.write(run_xml)\n</code></pre>"},{"location":"reference/flows/#openml.flows.assert_flows_equal","title":"<code>assert_flows_equal(flow1, flow2, ignore_parameter_values_on_older_children=None, ignore_parameter_values=False, ignore_custom_name_if_none=False, check_description=True)</code>","text":"<p>Check equality of two flows.</p> <p>Two flows are equal if their all keys which are not set by the server are equal, as well as all their parameters and components.</p> <p>Parameters:</p> Name Type Description Default <code>flow1</code> <code>OpenMLFlow</code> required <code>flow2</code> <code>OpenMLFlow</code> required <code>ignore_parameter_values_on_older_children</code> <code>str(optional)</code> <p>If set to <code>OpenMLFlow.upload_date</code>, ignores parameters in a child flow if it's upload date predates the upload date of the parent flow.</p> <code>None</code> <code>ignore_parameter_values</code> <code>bool</code> <p>Whether to ignore parameter values when comparing flows.</p> <code>False</code> <code>ignore_custom_name_if_none</code> <code>bool</code> <p>Whether to ignore the custom name field if either flow has <code>custom_name</code> equal to <code>None</code>.</p> <code>False</code> <code>check_description</code> <code>bool</code> <p>Whether to ignore matching of flow descriptions.</p> <code>True</code> Source code in <code>openml/flows/functions.py</code> <pre><code>def assert_flows_equal(  # noqa: C901, PLR0912, PLR0913, PLR0915\n    flow1: OpenMLFlow,\n    flow2: OpenMLFlow,\n    ignore_parameter_values_on_older_children: str | None = None,\n    ignore_parameter_values: bool = False,  # noqa: FBT001, FBT002\n    ignore_custom_name_if_none: bool = False,  # noqa:  FBT001, FBT002\n    check_description: bool = True,  # noqa:  FBT001, FBT002\n) -&gt; None:\n    \"\"\"Check equality of two flows.\n\n    Two flows are equal if their all keys which are not set by the server\n    are equal, as well as all their parameters and components.\n\n    Parameters\n    ----------\n    flow1 : OpenMLFlow\n\n    flow2 : OpenMLFlow\n\n    ignore_parameter_values_on_older_children : str (optional)\n        If set to ``OpenMLFlow.upload_date``, ignores parameters in a child\n        flow if it's upload date predates the upload date of the parent flow.\n\n    ignore_parameter_values : bool\n        Whether to ignore parameter values when comparing flows.\n\n    ignore_custom_name_if_none : bool\n        Whether to ignore the custom name field if either flow has `custom_name` equal to `None`.\n\n    check_description : bool\n        Whether to ignore matching of flow descriptions.\n    \"\"\"\n    if not isinstance(flow1, OpenMLFlow):\n        raise TypeError(\"Argument 1 must be of type OpenMLFlow, but is %s\" % type(flow1))\n\n    if not isinstance(flow2, OpenMLFlow):\n        raise TypeError(\"Argument 2 must be of type OpenMLFlow, but is %s\" % type(flow2))\n\n    # TODO as they are actually now saved during publish, it might be good to\n    # check for the equality of these as well.\n    generated_by_the_server = [\n        \"flow_id\",\n        \"uploader\",\n        \"version\",\n        \"upload_date\",\n        # Tags aren't directly created by the server,\n        # but the uploader has no control over them!\n        \"tags\",\n    ]\n    ignored_by_python_api = [\"binary_url\", \"binary_format\", \"binary_md5\", \"model\", \"_entity_id\"]\n\n    for key in set(flow1.__dict__.keys()).union(flow2.__dict__.keys()):\n        if key in generated_by_the_server + ignored_by_python_api:\n            continue\n        attr1 = getattr(flow1, key, None)\n        attr2 = getattr(flow2, key, None)\n        if key == \"components\":\n            if not (isinstance(attr1, Dict) and isinstance(attr2, Dict)):\n                raise TypeError(\"Cannot compare components because they are not dictionary.\")\n\n            for name in set(attr1.keys()).union(attr2.keys()):\n                if name not in attr1:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                if name not in attr2:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                assert_flows_equal(\n                    attr1[name],\n                    attr2[name],\n                    ignore_parameter_values_on_older_children,\n                    ignore_parameter_values,\n                    ignore_custom_name_if_none,\n                )\n        elif key == \"_extension\":\n            continue\n        elif check_description and key == \"description\":\n            # to ignore matching of descriptions since sklearn based flows may have\n            # altering docstrings and is not guaranteed to be consistent\n            continue\n        else:\n            if key == \"parameters\":\n                if ignore_parameter_values or ignore_parameter_values_on_older_children:\n                    params_flow_1 = set(flow1.parameters.keys())\n                    params_flow_2 = set(flow2.parameters.keys())\n                    symmetric_difference = params_flow_1 ^ params_flow_2\n                    if len(symmetric_difference) &gt; 0:\n                        raise ValueError(\n                            \"Flow %s: parameter set of flow \"\n                            \"differs from the parameters stored \"\n                            \"on the server.\" % flow1.name,\n                        )\n\n                if ignore_parameter_values_on_older_children:\n                    assert (\n                        flow1.upload_date is not None\n                    ), \"Flow1 has no upload date that allows us to compare age of children.\"\n                    upload_date_current_flow = dateutil.parser.parse(flow1.upload_date)\n                    upload_date_parent_flow = dateutil.parser.parse(\n                        ignore_parameter_values_on_older_children,\n                    )\n                    if upload_date_current_flow &lt; upload_date_parent_flow:\n                        continue\n\n                if ignore_parameter_values:\n                    # Continue needs to be done here as the first if\n                    # statement triggers in both special cases\n                    continue\n            elif (\n                key == \"custom_name\"\n                and ignore_custom_name_if_none\n                and (attr1 is None or attr2 is None)\n            ):\n                # If specified, we allow `custom_name` inequality if one flow's name is None.\n                # Helps with backwards compatibility as `custom_name` is now auto-generated, but\n                # before it used to be `None`.\n                continue\n            elif key == \"parameters_meta_info\":\n                # this value is a dictionary where each key is a parameter name, containing another\n                # dictionary with keys specifying the parameter's 'description' and 'data_type'\n                # checking parameter descriptions can be ignored since that might change\n                # data type check can also be ignored if one of them is not defined, i.e., None\n                params1 = set(flow1.parameters_meta_info)\n                params2 = set(flow2.parameters_meta_info)\n                if params1 != params2:\n                    raise ValueError(\n                        \"Parameter list in meta info for parameters differ \" \"in the two flows.\",\n                    )\n                # iterating over the parameter's meta info list\n                for param in params1:\n                    if (\n                        isinstance(flow1.parameters_meta_info[param], Dict)\n                        and isinstance(flow2.parameters_meta_info[param], Dict)\n                        and \"data_type\" in flow1.parameters_meta_info[param]\n                        and \"data_type\" in flow2.parameters_meta_info[param]\n                    ):\n                        value1 = flow1.parameters_meta_info[param][\"data_type\"]\n                        value2 = flow2.parameters_meta_info[param][\"data_type\"]\n                    else:\n                        value1 = flow1.parameters_meta_info[param]\n                        value2 = flow2.parameters_meta_info[param]\n                    if value1 is None or value2 is None:\n                        continue\n\n                    if value1 != value2:\n                        raise ValueError(\n                            f\"Flow {flow1.name}: data type for parameter {param} in {key} differ \"\n                            f\"as {value1}\\nvs\\n{value2}\",\n                        )\n                # the continue is to avoid the 'attr != attr2' check at end of function\n                continue\n\n            if attr1 != attr2:\n                raise ValueError(\n                    f\"Flow {flow1.name!s}: values for attribute '{key!s}' differ: \"\n                    f\"'{attr1!s}'\\nvs\\n'{attr2!s}'.\",\n                )\n</code></pre>"},{"location":"reference/flows/#openml.flows.delete_flow","title":"<code>delete_flow(flow_id)</code>","text":"<p>Delete flow with id <code>flow_id</code> from the OpenML server.</p> <p>You can only delete flows which you uploaded and which which are not linked to runs.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>OpenML id of the flow</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def delete_flow(flow_id: int) -&gt; bool:\n    \"\"\"Delete flow with id `flow_id` from the OpenML server.\n\n    You can only delete flows which you uploaded and which\n    which are not linked to runs.\n\n    Parameters\n    ----------\n    flow_id : int\n        OpenML id of the flow\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"flow\", flow_id)\n</code></pre>"},{"location":"reference/flows/#openml.flows.flow_exists","title":"<code>flow_exists(name, external_version)</code>","text":"<p>Retrieves the flow id.</p> <p>A flow is uniquely identified by name + external_version.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>string</code> <p>Name of the flow</p> required <code>external_version</code> <code>string</code> <p>Version information associated with flow.</p> required <p>Returns:</p> Name Type Description <code>flow_exist</code> <code>int or bool</code> <p>flow id iff exists, False otherwise</p> Notes <p>see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def flow_exists(name: str, external_version: str) -&gt; int | bool:\n    \"\"\"Retrieves the flow id.\n\n    A flow is uniquely identified by name + external_version.\n\n    Parameters\n    ----------\n    name : string\n        Name of the flow\n    external_version : string\n        Version information associated with flow.\n\n    Returns\n    -------\n    flow_exist : int or bool\n        flow id iff exists, False otherwise\n\n    Notes\n    -----\n    see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version\n    \"\"\"\n    if not (isinstance(name, str) and len(name) &gt; 0):\n        raise ValueError(\"Argument 'name' should be a non-empty string\")\n    if not (isinstance(name, str) and len(external_version) &gt; 0):\n        raise ValueError(\"Argument 'version' should be a non-empty string\")\n\n    xml_response = openml._api_calls._perform_api_call(\n        \"flow/exists\",\n        \"post\",\n        data={\"name\": name, \"external_version\": external_version},\n    )\n\n    result_dict = xmltodict.parse(xml_response)\n    flow_id = int(result_dict[\"oml:flow_exists\"][\"oml:id\"])\n    return flow_id if flow_id &gt; 0 else False\n</code></pre>"},{"location":"reference/flows/#openml.flows.get_flow","title":"<code>get_flow(flow_id, reinstantiate=False, strict_version=True)</code>","text":"<p>Download the OpenML flow for a given flow ID.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>The OpenML flow id.</p> required <code>reinstantiate</code> <code>bool</code> <p>Whether to reinstantiate the flow to a model instance.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>flow</code> <code>OpenMLFlow</code> <p>the flow</p> Source code in <code>openml/flows/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_flow(flow_id: int, reinstantiate: bool = False, strict_version: bool = True) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Download the OpenML flow for a given flow ID.\n\n    Parameters\n    ----------\n    flow_id : int\n        The OpenML flow id.\n\n    reinstantiate: bool\n        Whether to reinstantiate the flow to a model instance.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    flow : OpenMLFlow\n        the flow\n    \"\"\"\n    flow_id = int(flow_id)\n    flow = _get_flow_description(flow_id)\n\n    if reinstantiate:\n        flow.model = flow.extension.flow_to_model(flow, strict_version=strict_version)\n        if not strict_version:\n            # check if we need to return a new flow b/c of version mismatch\n            new_flow = flow.extension.model_to_flow(flow.model)\n            if new_flow.dependencies != flow.dependencies:\n                return new_flow\n    return flow\n</code></pre>"},{"location":"reference/flows/#openml.flows.get_flow_id","title":"<code>get_flow_id(model=None, name=None, exact_version=True)</code>","text":"<p>Retrieves the flow id for a model or a flow name.</p> <p>Provide either a model or a name to this function. Depending on the input, it does</p> <ul> <li><code>model</code> and <code>exact_version == True</code>: This helper function first queries for the necessary   extension. Second, it uses that extension to convert the model into a flow. Third, it   executes <code>flow_exists</code> to potentially obtain the flow id the flow is published to the   server.</li> <li><code>model</code> and <code>exact_version == False</code>: This helper function first queries for the   necessary extension. Second, it uses that extension to convert the model into a flow. Third   it calls <code>list_flows</code> and filters the returned values based on the flow name.</li> <li><code>name</code>: Ignores <code>exact_version</code> and calls <code>list_flows</code>, then filters the returned   values based on the flow name.</li> </ul> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>object</code> <p>Any model. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>name</code> <code>str</code> <p>Name of the flow. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>exact_version</code> <code>bool</code> <p>Whether to return the flow id of the exact version or all flow ids where the name of the flow matches. This is only taken into account for a model where a version number is available (requires <code>model</code> to be set).</p> <code>True</code> <p>Returns:</p> Type Description <code>(int or bool, List)</code> <p>flow id iff exists, <code>False</code> otherwise, List if <code>exact_version is False</code></p> Source code in <code>openml/flows/functions.py</code> <pre><code>def get_flow_id(\n    model: Any | None = None,\n    name: str | None = None,\n    exact_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; int | bool | list[int]:\n    \"\"\"Retrieves the flow id for a model or a flow name.\n\n    Provide either a model or a name to this function. Depending on the input, it does\n\n    * ``model`` and ``exact_version == True``: This helper function first queries for the necessary\n      extension. Second, it uses that extension to convert the model into a flow. Third, it\n      executes ``flow_exists`` to potentially obtain the flow id the flow is published to the\n      server.\n    * ``model`` and ``exact_version == False``: This helper function first queries for the\n      necessary extension. Second, it uses that extension to convert the model into a flow. Third\n      it calls ``list_flows`` and filters the returned values based on the flow name.\n    * ``name``: Ignores ``exact_version`` and calls ``list_flows``, then filters the returned\n      values based on the flow name.\n\n    Parameters\n    ----------\n    model : object\n        Any model. Must provide either ``model`` or ``name``.\n    name : str\n        Name of the flow. Must provide either ``model`` or ``name``.\n    exact_version : bool\n        Whether to return the flow id of the exact version or all flow ids where the name\n        of the flow matches. This is only taken into account for a model where a version number\n        is available (requires ``model`` to be set).\n\n    Returns\n    -------\n    int or bool, List\n        flow id iff exists, ``False`` otherwise, List if ``exact_version is False``\n    \"\"\"\n    if model is not None and name is not None:\n        raise ValueError(\"Must provide either argument `model` or argument `name`, but not both.\")\n\n    if model is not None:\n        extension = openml.extensions.get_extension_by_model(model, raise_if_no_extension=True)\n        if extension is None:\n            # This should never happen and is only here to please mypy will be gone soon once the\n            # whole function is removed\n            raise TypeError(extension)\n        flow = extension.model_to_flow(model)\n        flow_name = flow.name\n        external_version = flow.external_version\n    elif name is not None:\n        flow_name = name\n        exact_version = False\n        external_version = None\n    else:\n        raise ValueError(\n            \"Need to provide either argument `model` or argument `name`, but both are `None`.\"\n        )\n\n    if exact_version:\n        if external_version is None:\n            raise ValueError(\"exact_version should be False if model is None!\")\n        return flow_exists(name=flow_name, external_version=external_version)\n\n    flows = list_flows(output_format=\"dataframe\")\n    assert isinstance(flows, pd.DataFrame)  # Make mypy happy\n    flows = flows.query(f'name == \"{flow_name}\"')\n    return flows[\"id\"].to_list()  # type: ignore[no-any-return]\n</code></pre>"},{"location":"reference/flows/#openml.flows.list_flows","title":"<code>list_flows(offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; dict\n</code></pre><pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_flows(offset: int | None, size: int | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all flows which are on OpenML. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of flows to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of flows to return</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: uploader.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>flows</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from flow_id to a dict giving a brief overview of the     respective flow.     Every flow is represented by a dictionary containing     the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> </ul> Source code in <code>openml/flows/functions.py</code> <pre><code>def list_flows(\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all flows which are on OpenML.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of flows to skip, starting from the first\n    size : int, optional\n        the maximum number of flows to return\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: uploader.\n\n    Returns\n    -------\n    flows : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from flow_id to a dict giving a brief overview of the\n            respective flow.\n            Every flow is represented by a dictionary containing\n            the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(\n        list_output_format=output_format,\n        listing_call=_list_flows,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/flows/flow/","title":"flow","text":""},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow","title":"<code>OpenMLFlow</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Flow. Stores machine learning models.</p> <p>Flows should not be generated manually, but by the function :meth:<code>openml.flows.create_flow_from_model</code>. Using this helper function ensures that all relevant fields are filled in.</p> <p>Implements <code>openml.implementation.upload.xsd &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/ openml.implementation.upload.xsd&gt;</code>_.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name of the flow. Is used together with the attribute <code>external_version</code> as a unique identifier of the flow.</p> required <code>description</code> <code>str</code> <p>Human-readable description of the flow (free text).</p> required <code>model</code> <code>object</code> <p>ML model which is described by this flow.</p> required <code>components</code> <code>OrderedDict</code> <p>Mapping from component identifier to an OpenMLFlow object. Components are usually subfunctions of an algorithm (e.g. kernels), base learners in ensemble algorithms (decision tree in adaboost) or building blocks of a machine learning pipeline. Components are modeled as independent flows and can be shared between flows (different pipelines can use the same components).</p> required <code>parameters</code> <code>OrderedDict</code> <p>Mapping from parameter name to the parameter default value. The parameter default value must be of type <code>str</code>, so that the respective toolbox plugin can take care of casting the parameter default value to the correct type.</p> required <code>parameters_meta_info</code> <code>OrderedDict</code> <p>Mapping from parameter name to <code>dict</code>. Stores additional information for each parameter. Required keys are <code>data_type</code> and <code>description</code>.</p> required <code>external_version</code> <code>str</code> <p>Version number of the software the flow is implemented in. Is used together with the attribute <code>name</code> as a uniquer identifier of the flow.</p> required <code>tags</code> <code>list</code> <p>List of tags. Created on the server by other API calls.</p> required <code>language</code> <code>str</code> <p>Natural language the flow is described in (not the programming language).</p> required <code>dependencies</code> <code>str</code> <p>A list of dependencies necessary to run the flow. This field should contain all libraries the flow depends on. To allow reproducibility it should also specify the exact version numbers.</p> required <code>class_name</code> <code>str</code> <p>The development language name of the class which is described by this flow.</p> <code>None</code> <code>custom_name</code> <code>str</code> <p>Custom name of the flow given by the owner.</p> <code>None</code> <code>binary_url</code> <code>str</code> <p>Url from which the binary can be downloaded. Added by the server. Ignored when uploaded manually. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_format</code> <code>str</code> <p>Format in which the binary code was uploaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>binary_md5</code> <code>str</code> <p>MD5 checksum to check if the binary code was correctly downloaded. Will not be used by the python API because binaries aren't compatible across machines.</p> <code>None</code> <code>uploader</code> <code>str</code> <p>OpenML user ID of the uploader. Filled in by the server.</p> <code>None</code> <code>upload_date</code> <code>str</code> <p>Date the flow was uploaded. Filled in by the server.</p> <code>None</code> <code>flow_id</code> <code>int</code> <p>Flow ID. Assigned by the server.</p> <code>None</code> <code>extension</code> <code>Extension</code> <p>The extension for a flow (e.g., sklearn).</p> <code>None</code> <code>version</code> <code>str</code> <p>OpenML version of the flow. Assigned by the server.</p> <code>None</code> Source code in <code>openml/flows/flow.py</code> <pre><code>class OpenMLFlow(OpenMLBase):\n    \"\"\"OpenML Flow. Stores machine learning models.\n\n    Flows should not be generated manually, but by the function\n    :meth:`openml.flows.create_flow_from_model`. Using this helper function\n    ensures that all relevant fields are filled in.\n\n    Implements `openml.implementation.upload.xsd\n    &lt;https://github.com/openml/openml/blob/master/openml_OS/views/pages/api_new/v1/xsd/\n    openml.implementation.upload.xsd&gt;`_.\n\n    Parameters\n    ----------\n    name : str\n        Name of the flow. Is used together with the attribute\n        `external_version` as a unique identifier of the flow.\n    description : str\n        Human-readable description of the flow (free text).\n    model : object\n        ML model which is described by this flow.\n    components : OrderedDict\n        Mapping from component identifier to an OpenMLFlow object. Components\n        are usually subfunctions of an algorithm (e.g. kernels), base learners\n        in ensemble algorithms (decision tree in adaboost) or building blocks\n        of a machine learning pipeline. Components are modeled as independent\n        flows and can be shared between flows (different pipelines can use\n        the same components).\n    parameters : OrderedDict\n        Mapping from parameter name to the parameter default value. The\n        parameter default value must be of type `str`, so that the respective\n        toolbox plugin can take care of casting the parameter default value to\n        the correct type.\n    parameters_meta_info : OrderedDict\n        Mapping from parameter name to `dict`. Stores additional information\n        for each parameter. Required keys are `data_type` and `description`.\n    external_version : str\n        Version number of the software the flow is implemented in. Is used\n        together with the attribute `name` as a uniquer identifier of the flow.\n    tags : list\n        List of tags. Created on the server by other API calls.\n    language : str\n        Natural language the flow is described in (not the programming\n        language).\n    dependencies : str\n        A list of dependencies necessary to run the flow. This field should\n        contain all libraries the flow depends on. To allow reproducibility\n        it should also specify the exact version numbers.\n    class_name : str, optional\n        The development language name of the class which is described by this\n        flow.\n    custom_name : str, optional\n        Custom name of the flow given by the owner.\n    binary_url : str, optional\n        Url from which the binary can be downloaded. Added by the server.\n        Ignored when uploaded manually. Will not be used by the python API\n        because binaries aren't compatible across machines.\n    binary_format : str, optional\n        Format in which the binary code was uploaded. Will not be used by the\n        python API because binaries aren't compatible across machines.\n    binary_md5 : str, optional\n        MD5 checksum to check if the binary code was correctly downloaded. Will\n        not be used by the python API because binaries aren't compatible across\n        machines.\n    uploader : str, optional\n        OpenML user ID of the uploader. Filled in by the server.\n    upload_date : str, optional\n        Date the flow was uploaded. Filled in by the server.\n    flow_id : int, optional\n        Flow ID. Assigned by the server.\n    extension : Extension, optional\n        The extension for a flow (e.g., sklearn).\n    version : str, optional\n        OpenML version of the flow. Assigned by the server.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        name: str,\n        description: str,\n        model: object,\n        components: dict,\n        parameters: dict,\n        parameters_meta_info: dict,\n        external_version: str,\n        tags: list,\n        language: str,\n        dependencies: str,\n        class_name: str | None = None,\n        custom_name: str | None = None,\n        binary_url: str | None = None,\n        binary_format: str | None = None,\n        binary_md5: str | None = None,\n        uploader: str | None = None,\n        upload_date: str | None = None,\n        flow_id: int | None = None,\n        extension: Extension | None = None,\n        version: str | None = None,\n    ):\n        self.name = name\n        self.description = description\n        self.model = model\n\n        for variable, variable_name in [\n            [components, \"components\"],\n            [parameters, \"parameters\"],\n            [parameters_meta_info, \"parameters_meta_info\"],\n        ]:\n            if not isinstance(variable, (OrderedDict, dict)):\n                raise TypeError(\n                    f\"{variable_name} must be of type OrderedDict or dict, \"\n                    f\"but is {type(variable)}.\",\n                )\n\n        self.components = components\n        self.parameters = parameters\n        self.parameters_meta_info = parameters_meta_info\n        self.class_name = class_name\n\n        keys_parameters = set(parameters.keys())\n        keys_parameters_meta_info = set(parameters_meta_info.keys())\n        if len(keys_parameters.difference(keys_parameters_meta_info)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters, but not in \"\n                \"parameters_meta_info.\"\n                % str(keys_parameters.difference(keys_parameters_meta_info)),\n            )\n        if len(keys_parameters_meta_info.difference(keys_parameters)) &gt; 0:\n            raise ValueError(\n                \"Parameter %s only in parameters_meta_info, \"\n                \"but not in parameters.\"\n                % str(keys_parameters_meta_info.difference(keys_parameters)),\n            )\n\n        self.external_version = external_version\n        self.uploader = uploader\n\n        self.custom_name = custom_name\n        self.tags = tags if tags is not None else []\n        self.binary_url = binary_url\n        self.binary_format = binary_format\n        self.binary_md5 = binary_md5\n        self.version = version\n        self.upload_date = upload_date\n        self.language = language\n        self.dependencies = dependencies\n        self.flow_id = flow_id\n        if extension is None:\n            self._extension = get_extension_by_flow(self)\n        else:\n            self._extension = extension\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the flow.\"\"\"\n        return self.flow_id\n\n    @property\n    def extension(self) -&gt; Extension:\n        \"\"\"The extension of the flow (e.g., sklearn).\"\"\"\n        if self._extension is not None:\n            return self._extension\n\n        raise RuntimeError(\n            f\"No extension could be found for flow {self.flow_id}: {self.name}\",\n        )\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields = {\n            \"Flow Name\": self.name,\n            \"Flow Description\": self.description,\n            \"Dependencies\": self.dependencies,\n        }\n        if self.flow_id is not None:\n            fields[\"Flow URL\"] = self.openml_url if self.openml_url is not None else \"None\"\n            fields[\"Flow ID\"] = str(self.flow_id)\n            if self.version is not None:\n                fields[\"Flow ID\"] += f\" (version {self.version})\"\n        if self.upload_date is not None:\n            fields[\"Upload Date\"] = self.upload_date.replace(\"T\", \" \")\n        if self.binary_url is not None:\n            fields[\"Binary URL\"] = self.binary_url\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Flow ID\",\n            \"Flow URL\",\n            \"Flow Name\",\n            \"Flow Description\",\n            \"Binary URL\",\n            \"Upload Date\",\n            \"Dependencies\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: C901, PLR0912\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        flow_container = OrderedDict()  # type: 'dict[str, dict]'\n        flow_dict = OrderedDict(\n            [(\"@xmlns:oml\", \"http://openml.org/openml\")],\n        )  # type: 'dict[str, list | str]'  # E501\n        flow_container[\"oml:flow\"] = flow_dict\n        _add_if_nonempty(flow_dict, \"oml:id\", self.flow_id)\n\n        for required in [\"name\", \"external_version\"]:\n            if getattr(self, required) is None:\n                raise ValueError(f\"self.{required} is required but None\")\n        for attribute in [\n            \"uploader\",\n            \"name\",\n            \"custom_name\",\n            \"class_name\",\n            \"version\",\n            \"external_version\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n        ]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        if not self.description:\n            logger = logging.getLogger(__name__)\n            logger.warning(\"Flow % has empty description\", self.name)\n\n        flow_parameters = []\n        for key in self.parameters:\n            param_dict = OrderedDict()  # type: 'OrderedDict[str, str]'\n            param_dict[\"oml:name\"] = key\n            meta_info = self.parameters_meta_info[key]\n\n            _add_if_nonempty(param_dict, \"oml:data_type\", meta_info[\"data_type\"])\n            param_dict[\"oml:default_value\"] = self.parameters[key]\n            _add_if_nonempty(param_dict, \"oml:description\", meta_info[\"description\"])\n\n            for key_, value in param_dict.items():\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n                if value is not None and not isinstance(value, str):\n                    raise ValueError(\n                        f\"Parameter value {value} cannot be serialized \"\n                        f\"because it is of type {type(value)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            flow_parameters.append(param_dict)\n\n        flow_dict[\"oml:parameter\"] = flow_parameters\n\n        components = []\n        for key in self.components:\n            component_dict = OrderedDict()  # type: 'OrderedDict[str, dict]'\n            component_dict[\"oml:identifier\"] = key\n            if self.components[key] in [\"passthrough\", \"drop\"]:\n                component_dict[\"oml:flow\"] = {\n                    \"oml-python:serialized_object\": \"component_reference\",\n                    \"value\": {\"key\": self.components[key], \"step_name\": self.components[key]},\n                }\n            else:\n                component_dict[\"oml:flow\"] = self.components[key]._to_dict()[\"oml:flow\"]\n\n            for key_ in component_dict:\n                # We only need to check if the key is a string, because the\n                # value is a flow. The flow itself is valid by recursion\n                if key_ is not None and not isinstance(key_, str):\n                    raise ValueError(\n                        f\"Parameter name {key_} cannot be serialized \"\n                        f\"because it is of type {type(key_)}. Only strings \"\n                        \"can be serialized.\",\n                    )\n\n            components.append(component_dict)\n\n        flow_dict[\"oml:component\"] = components\n        flow_dict[\"oml:tag\"] = self.tags\n        for attribute in [\"binary_url\", \"binary_format\", \"binary_md5\"]:\n            _add_if_nonempty(flow_dict, f\"oml:{attribute}\", getattr(self, attribute))\n\n        return flow_container\n\n    @classmethod\n    def _from_dict(cls, xml_dict: dict) -&gt; OpenMLFlow:\n        \"\"\"Create a flow from an xml description.\n\n        Calls itself recursively to create :class:`OpenMLFlow` objects of\n        subflows (components).\n\n        XML definition of a flow is available at\n        https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd\n\n        Parameters\n        ----------\n        xml_dict : dict\n            Dictionary representation of the flow as created by _to_dict()\n\n        Returns\n        -------\n            OpenMLFlow\n\n        \"\"\"  # E501\n        arguments = OrderedDict()\n        dic = xml_dict[\"oml:flow\"]\n\n        # Mandatory parts in the xml file\n        for key in [\"name\"]:\n            arguments[key] = dic[\"oml:\" + key]\n\n        # non-mandatory parts in the xml file\n        for key in [\n            \"external_version\",\n            \"uploader\",\n            \"description\",\n            \"upload_date\",\n            \"language\",\n            \"dependencies\",\n            \"version\",\n            \"binary_url\",\n            \"binary_format\",\n            \"binary_md5\",\n            \"class_name\",\n            \"custom_name\",\n        ]:\n            arguments[key] = dic.get(\"oml:\" + key)\n\n        # has to be converted to an int if present and cannot parsed in the\n        # two loops above\n        arguments[\"flow_id\"] = int(dic[\"oml:id\"]) if dic.get(\"oml:id\") is not None else None\n\n        # Now parse parts of a flow which can occur multiple times like\n        # parameters, components (subflows) and tags. These can't be tackled\n        # in the loops above because xmltodict returns a dict if such an\n        # entity occurs once, and a list if it occurs multiple times.\n        # Furthermore, they must be treated differently, for example\n        # for components this method is called recursively and\n        # for parameters the actual information is split into two dictionaries\n        # for easier access in python.\n\n        parameters = OrderedDict()\n        parameters_meta_info = OrderedDict()\n        if \"oml:parameter\" in dic:\n            # In case of a single parameter, xmltodict returns a dictionary,\n            # otherwise a list.\n            oml_parameters = extract_xml_tags(\"oml:parameter\", dic, allow_none=False)\n\n            for oml_parameter in oml_parameters:\n                parameter_name = oml_parameter[\"oml:name\"]\n                default_value = oml_parameter[\"oml:default_value\"]\n                parameters[parameter_name] = default_value\n\n                meta_info = OrderedDict()\n                meta_info[\"description\"] = oml_parameter.get(\"oml:description\")\n                meta_info[\"data_type\"] = oml_parameter.get(\"oml:data_type\")\n                parameters_meta_info[parameter_name] = meta_info\n        arguments[\"parameters\"] = parameters\n        arguments[\"parameters_meta_info\"] = parameters_meta_info\n\n        components = OrderedDict()\n        if \"oml:component\" in dic:\n            # In case of a single component xmltodict returns a dict,\n            # otherwise a list.\n            oml_components = extract_xml_tags(\"oml:component\", dic, allow_none=False)\n\n            for component in oml_components:\n                flow = OpenMLFlow._from_dict(component)\n                components[component[\"oml:identifier\"]] = flow\n        arguments[\"components\"] = components\n        arguments[\"tags\"] = extract_xml_tags(\"oml:tag\", dic)\n\n        arguments[\"model\"] = None\n        return cls(**arguments)\n\n    def to_filesystem(self, output_directory: str | Path) -&gt; None:\n        \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n        output_directory = Path(output_directory)\n        output_directory.mkdir(parents=True, exist_ok=True)\n\n        output_path = output_directory / \"flow.xml\"\n        if output_path.exists():\n            raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n        run_xml = self._to_xml()\n        with output_path.open(\"w\") as f:\n            f.write(run_xml)\n\n    @classmethod\n    def from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n        \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n        input_directory = Path(input_directory) / \"flow.xml\"\n        with input_directory.open() as f:\n            xml_string = f.read()\n        return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.flow_id = int(xml_response[\"oml:upload_flow\"][\"oml:id\"])\n\n    def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n        \"\"\"Publish this flow to OpenML server.\n\n        Raises a PyOpenMLError if the flow exists on the server, but\n        `self.flow_id` does not match the server known flow id.\n\n        Parameters\n        ----------\n        raise_error_if_exists : bool, optional (default=False)\n            If True, raise PyOpenMLError if the flow exists on the server.\n            If False, update the local flow to match the server flow.\n\n        Returns\n        -------\n        self : OpenMLFlow\n\n        \"\"\"\n        # Import at top not possible because of cyclic dependencies. In\n        # particular, flow.py tries to import functions.py in order to call\n        # get_flow(), while functions.py tries to import flow.py in order to\n        # instantiate an OpenMLFlow.\n        import openml.flows.functions\n\n        flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n        if not flow_id:\n            if self.flow_id:\n                raise openml.exceptions.PyOpenMLError(\n                    \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n                )\n            super().publish()\n            assert self.flow_id is not None  # for mypy\n            flow_id = self.flow_id\n        elif raise_error_if_exists:\n            error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n            raise openml.exceptions.PyOpenMLError(error_message)\n        elif self.flow_id is not None and self.flow_id != flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n            )\n\n        flow = openml.flows.functions.get_flow(flow_id)\n        _copy_server_fields(flow, self)\n        try:\n            openml.flows.functions.assert_flows_equal(\n                self,\n                flow,\n                flow.upload_date,\n                ignore_parameter_values=True,\n                ignore_custom_name_if_none=True,\n            )\n        except ValueError as e:\n            message = e.args[0]\n            raise ValueError(\n                \"The flow on the server is inconsistent with the local flow. \"\n                f\"The server flow ID is {flow_id}. Please check manually and remove \"\n                f\"the flow if necessary! Error is:\\n'{message}'\",\n            ) from e\n        return self\n\n    def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n        \"\"\"\n        Returns for each sub-component of the flow the path of identifiers\n        that should be traversed to reach this component. The resulting dict\n        maps a key (identifying a flow by either its id, name or fullname) to\n        the parameter prefix.\n\n        Parameters\n        ----------\n        key_item: str\n            The flow attribute that will be used to identify flows in the\n            structure. Allowed values {flow_id, name}\n\n        Returns\n        -------\n        dict[str, List[str]]\n            The flow structure\n        \"\"\"\n        if key_item not in [\"flow_id\", \"name\"]:\n            raise ValueError(\"key_item should be in {flow_id, name}\")\n        structure = {}\n        for key, sub_flow in self.components.items():\n            sub_structure = sub_flow.get_structure(key_item)\n            for flow_name, flow_sub_structure in sub_structure.items():\n                structure[flow_name] = [key, *flow_sub_structure]\n        structure[getattr(self, key_item)] = []\n        return structure\n\n    def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n        \"\"\"\n        Returns a subflow from the tree of dependencies.\n\n        Parameters\n        ----------\n        structure: list[str]\n            A list of strings, indicating the location of the subflow\n\n        Returns\n        -------\n        OpenMLFlow\n            The OpenMLFlow that corresponds to the structure\n        \"\"\"\n        # make a copy of structure, as we don't want to change it in the\n        # outer scope\n        structure = list(structure)\n        if len(structure) &lt; 1:\n            raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n        sub_identifier = structure[0]\n        if sub_identifier not in self.components:\n            raise ValueError(\n                f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n            )\n        if len(structure) == 1:\n            return self.components[sub_identifier]  # type: ignore\n\n        structure.pop(0)\n        return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.extension","title":"<code>extension: Extension</code>  <code>property</code>","text":"<p>The extension of the flow (e.g., sklearn).</p>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the flow.</p>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.from_filesystem","title":"<code>from_filesystem(input_directory)</code>  <code>classmethod</code>","text":"<p>Read a flow from an XML in input_directory on the filesystem.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, input_directory: str | Path) -&gt; OpenMLFlow:\n    \"\"\"Read a flow from an XML in input_directory on the filesystem.\"\"\"\n    input_directory = Path(input_directory) / \"flow.xml\"\n    with input_directory.open() as f:\n        xml_string = f.read()\n    return OpenMLFlow._from_dict(xmltodict.parse(xml_string))\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.get_structure","title":"<code>get_structure(key_item)</code>","text":"<p>Returns for each sub-component of the flow the path of identifiers that should be traversed to reach this component. The resulting dict maps a key (identifying a flow by either its id, name or fullname) to the parameter prefix.</p> <p>Parameters:</p> Name Type Description Default <code>key_item</code> <code>str</code> <p>The flow attribute that will be used to identify flows in the structure. Allowed values {flow_id, name}</p> required <p>Returns:</p> Type Description <code>dict[str, List[str]]</code> <p>The flow structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_structure(self, key_item: str) -&gt; dict[str, list[str]]:\n    \"\"\"\n    Returns for each sub-component of the flow the path of identifiers\n    that should be traversed to reach this component. The resulting dict\n    maps a key (identifying a flow by either its id, name or fullname) to\n    the parameter prefix.\n\n    Parameters\n    ----------\n    key_item: str\n        The flow attribute that will be used to identify flows in the\n        structure. Allowed values {flow_id, name}\n\n    Returns\n    -------\n    dict[str, List[str]]\n        The flow structure\n    \"\"\"\n    if key_item not in [\"flow_id\", \"name\"]:\n        raise ValueError(\"key_item should be in {flow_id, name}\")\n    structure = {}\n    for key, sub_flow in self.components.items():\n        sub_structure = sub_flow.get_structure(key_item)\n        for flow_name, flow_sub_structure in sub_structure.items():\n            structure[flow_name] = [key, *flow_sub_structure]\n    structure[getattr(self, key_item)] = []\n    return structure\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.get_subflow","title":"<code>get_subflow(structure)</code>","text":"<p>Returns a subflow from the tree of dependencies.</p> <p>Parameters:</p> Name Type Description Default <code>structure</code> <code>list[str]</code> <p>A list of strings, indicating the location of the subflow</p> required <p>Returns:</p> Type Description <code>OpenMLFlow</code> <p>The OpenMLFlow that corresponds to the structure</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def get_subflow(self, structure: list[str]) -&gt; OpenMLFlow:\n    \"\"\"\n    Returns a subflow from the tree of dependencies.\n\n    Parameters\n    ----------\n    structure: list[str]\n        A list of strings, indicating the location of the subflow\n\n    Returns\n    -------\n    OpenMLFlow\n        The OpenMLFlow that corresponds to the structure\n    \"\"\"\n    # make a copy of structure, as we don't want to change it in the\n    # outer scope\n    structure = list(structure)\n    if len(structure) &lt; 1:\n        raise ValueError(\"Please provide a structure list of size &gt;= 1\")\n    sub_identifier = structure[0]\n    if sub_identifier not in self.components:\n        raise ValueError(\n            f\"Flow {self.name} does not contain component with \" f\"identifier {sub_identifier}\",\n        )\n    if len(structure) == 1:\n        return self.components[sub_identifier]  # type: ignore\n\n    structure.pop(0)\n    return self.components[sub_identifier].get_subflow(structure)  # type: ignore\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.publish","title":"<code>publish(raise_error_if_exists=False)</code>","text":"<p>Publish this flow to OpenML server.</p> <p>Raises a PyOpenMLError if the flow exists on the server, but <code>self.flow_id</code> does not match the server known flow id.</p> <p>Parameters:</p> Name Type Description Default <code>raise_error_if_exists</code> <code>(bool, optional(default=False))</code> <p>If True, raise PyOpenMLError if the flow exists on the server. If False, update the local flow to match the server flow.</p> <code>False</code> <p>Returns:</p> Name Type Description <code>self</code> <code>OpenMLFlow</code> Source code in <code>openml/flows/flow.py</code> <pre><code>def publish(self, raise_error_if_exists: bool = False) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Publish this flow to OpenML server.\n\n    Raises a PyOpenMLError if the flow exists on the server, but\n    `self.flow_id` does not match the server known flow id.\n\n    Parameters\n    ----------\n    raise_error_if_exists : bool, optional (default=False)\n        If True, raise PyOpenMLError if the flow exists on the server.\n        If False, update the local flow to match the server flow.\n\n    Returns\n    -------\n    self : OpenMLFlow\n\n    \"\"\"\n    # Import at top not possible because of cyclic dependencies. In\n    # particular, flow.py tries to import functions.py in order to call\n    # get_flow(), while functions.py tries to import flow.py in order to\n    # instantiate an OpenMLFlow.\n    import openml.flows.functions\n\n    flow_id = openml.flows.functions.flow_exists(self.name, self.external_version)\n    if not flow_id:\n        if self.flow_id:\n            raise openml.exceptions.PyOpenMLError(\n                \"Flow does not exist on the server, \" \"but 'flow.flow_id' is not None.\",\n            )\n        super().publish()\n        assert self.flow_id is not None  # for mypy\n        flow_id = self.flow_id\n    elif raise_error_if_exists:\n        error_message = f\"This OpenMLFlow already exists with id: {flow_id}.\"\n        raise openml.exceptions.PyOpenMLError(error_message)\n    elif self.flow_id is not None and self.flow_id != flow_id:\n        raise openml.exceptions.PyOpenMLError(\n            \"Local flow_id does not match server flow_id: \" f\"'{self.flow_id}' vs '{flow_id}'\",\n        )\n\n    flow = openml.flows.functions.get_flow(flow_id)\n    _copy_server_fields(flow, self)\n    try:\n        openml.flows.functions.assert_flows_equal(\n            self,\n            flow,\n            flow.upload_date,\n            ignore_parameter_values=True,\n            ignore_custom_name_if_none=True,\n        )\n    except ValueError as e:\n        message = e.args[0]\n        raise ValueError(\n            \"The flow on the server is inconsistent with the local flow. \"\n            f\"The server flow ID is {flow_id}. Please check manually and remove \"\n            f\"the flow if necessary! Error is:\\n'{message}'\",\n        ) from e\n    return self\n</code></pre>"},{"location":"reference/flows/flow/#openml.flows.flow.OpenMLFlow.to_filesystem","title":"<code>to_filesystem(output_directory)</code>","text":"<p>Write a flow to the filesystem as XML to output_directory.</p> Source code in <code>openml/flows/flow.py</code> <pre><code>def to_filesystem(self, output_directory: str | Path) -&gt; None:\n    \"\"\"Write a flow to the filesystem as XML to output_directory.\"\"\"\n    output_directory = Path(output_directory)\n    output_directory.mkdir(parents=True, exist_ok=True)\n\n    output_path = output_directory / \"flow.xml\"\n    if output_path.exists():\n        raise ValueError(\"Output directory already contains a flow.xml file.\")\n\n    run_xml = self._to_xml()\n    with output_path.open(\"w\") as f:\n        f.write(run_xml)\n</code></pre>"},{"location":"reference/flows/functions/","title":"functions","text":""},{"location":"reference/flows/functions/#openml.flows.functions.__list_flows","title":"<code>__list_flows(api_call, output_format='dict')</code>","text":"<pre><code>__list_flows(api_call: str, output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>__list_flows(api_call: str, output_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Retrieve information about flows from OpenML API and parse it to a dictionary or a Pandas DataFrame.</p> <p>Parameters:</p> Name Type Description Default <code>api_call</code> <code>str</code> <p>Retrieves the information about flows.</p> required <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The output format.</p> <code>'dict'</code> <p>Returns:</p> Type Description <code>    The flows information in the specified output format.</code> Source code in <code>openml/flows/functions.py</code> <pre><code>def __list_flows(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\"] = \"dict\"\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Retrieve information about flows from OpenML API\n    and parse it to a dictionary or a Pandas DataFrame.\n\n    Parameters\n    ----------\n    api_call: str\n        Retrieves the information about flows.\n    output_format: str in {\"dict\", \"dataframe\"}\n        The output format.\n\n    Returns\n    -------\n        The flows information in the specified output format.\n    \"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    flows_dict = xmltodict.parse(xml_string, force_list=(\"oml:flow\",))\n\n    # Minimalistic check if the XML is useful\n    assert isinstance(flows_dict[\"oml:flows\"][\"oml:flow\"], list), type(flows_dict[\"oml:flows\"])\n    assert flows_dict[\"oml:flows\"][\"@xmlns:oml\"] == \"http://openml.org/openml\", flows_dict[\n        \"oml:flows\"\n    ][\"@xmlns:oml\"]\n\n    flows = {}\n    for flow_ in flows_dict[\"oml:flows\"][\"oml:flow\"]:\n        fid = int(flow_[\"oml:id\"])\n        flow = {\n            \"id\": fid,\n            \"full_name\": flow_[\"oml:full_name\"],\n            \"name\": flow_[\"oml:name\"],\n            \"version\": flow_[\"oml:version\"],\n            \"external_version\": flow_[\"oml:external_version\"],\n            \"uploader\": flow_[\"oml:uploader\"],\n        }\n        flows[fid] = flow\n\n    if output_format == \"dataframe\":\n        flows = pd.DataFrame.from_dict(flows, orient=\"index\")\n\n    return flows\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.assert_flows_equal","title":"<code>assert_flows_equal(flow1, flow2, ignore_parameter_values_on_older_children=None, ignore_parameter_values=False, ignore_custom_name_if_none=False, check_description=True)</code>","text":"<p>Check equality of two flows.</p> <p>Two flows are equal if their all keys which are not set by the server are equal, as well as all their parameters and components.</p> <p>Parameters:</p> Name Type Description Default <code>flow1</code> <code>OpenMLFlow</code> required <code>flow2</code> <code>OpenMLFlow</code> required <code>ignore_parameter_values_on_older_children</code> <code>str(optional)</code> <p>If set to <code>OpenMLFlow.upload_date</code>, ignores parameters in a child flow if it's upload date predates the upload date of the parent flow.</p> <code>None</code> <code>ignore_parameter_values</code> <code>bool</code> <p>Whether to ignore parameter values when comparing flows.</p> <code>False</code> <code>ignore_custom_name_if_none</code> <code>bool</code> <p>Whether to ignore the custom name field if either flow has <code>custom_name</code> equal to <code>None</code>.</p> <code>False</code> <code>check_description</code> <code>bool</code> <p>Whether to ignore matching of flow descriptions.</p> <code>True</code> Source code in <code>openml/flows/functions.py</code> <pre><code>def assert_flows_equal(  # noqa: C901, PLR0912, PLR0913, PLR0915\n    flow1: OpenMLFlow,\n    flow2: OpenMLFlow,\n    ignore_parameter_values_on_older_children: str | None = None,\n    ignore_parameter_values: bool = False,  # noqa: FBT001, FBT002\n    ignore_custom_name_if_none: bool = False,  # noqa:  FBT001, FBT002\n    check_description: bool = True,  # noqa:  FBT001, FBT002\n) -&gt; None:\n    \"\"\"Check equality of two flows.\n\n    Two flows are equal if their all keys which are not set by the server\n    are equal, as well as all their parameters and components.\n\n    Parameters\n    ----------\n    flow1 : OpenMLFlow\n\n    flow2 : OpenMLFlow\n\n    ignore_parameter_values_on_older_children : str (optional)\n        If set to ``OpenMLFlow.upload_date``, ignores parameters in a child\n        flow if it's upload date predates the upload date of the parent flow.\n\n    ignore_parameter_values : bool\n        Whether to ignore parameter values when comparing flows.\n\n    ignore_custom_name_if_none : bool\n        Whether to ignore the custom name field if either flow has `custom_name` equal to `None`.\n\n    check_description : bool\n        Whether to ignore matching of flow descriptions.\n    \"\"\"\n    if not isinstance(flow1, OpenMLFlow):\n        raise TypeError(\"Argument 1 must be of type OpenMLFlow, but is %s\" % type(flow1))\n\n    if not isinstance(flow2, OpenMLFlow):\n        raise TypeError(\"Argument 2 must be of type OpenMLFlow, but is %s\" % type(flow2))\n\n    # TODO as they are actually now saved during publish, it might be good to\n    # check for the equality of these as well.\n    generated_by_the_server = [\n        \"flow_id\",\n        \"uploader\",\n        \"version\",\n        \"upload_date\",\n        # Tags aren't directly created by the server,\n        # but the uploader has no control over them!\n        \"tags\",\n    ]\n    ignored_by_python_api = [\"binary_url\", \"binary_format\", \"binary_md5\", \"model\", \"_entity_id\"]\n\n    for key in set(flow1.__dict__.keys()).union(flow2.__dict__.keys()):\n        if key in generated_by_the_server + ignored_by_python_api:\n            continue\n        attr1 = getattr(flow1, key, None)\n        attr2 = getattr(flow2, key, None)\n        if key == \"components\":\n            if not (isinstance(attr1, Dict) and isinstance(attr2, Dict)):\n                raise TypeError(\"Cannot compare components because they are not dictionary.\")\n\n            for name in set(attr1.keys()).union(attr2.keys()):\n                if name not in attr1:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                if name not in attr2:\n                    raise ValueError(\n                        \"Component %s only available in \" \"argument2, but not in argument1.\" % name,\n                    )\n                assert_flows_equal(\n                    attr1[name],\n                    attr2[name],\n                    ignore_parameter_values_on_older_children,\n                    ignore_parameter_values,\n                    ignore_custom_name_if_none,\n                )\n        elif key == \"_extension\":\n            continue\n        elif check_description and key == \"description\":\n            # to ignore matching of descriptions since sklearn based flows may have\n            # altering docstrings and is not guaranteed to be consistent\n            continue\n        else:\n            if key == \"parameters\":\n                if ignore_parameter_values or ignore_parameter_values_on_older_children:\n                    params_flow_1 = set(flow1.parameters.keys())\n                    params_flow_2 = set(flow2.parameters.keys())\n                    symmetric_difference = params_flow_1 ^ params_flow_2\n                    if len(symmetric_difference) &gt; 0:\n                        raise ValueError(\n                            \"Flow %s: parameter set of flow \"\n                            \"differs from the parameters stored \"\n                            \"on the server.\" % flow1.name,\n                        )\n\n                if ignore_parameter_values_on_older_children:\n                    assert (\n                        flow1.upload_date is not None\n                    ), \"Flow1 has no upload date that allows us to compare age of children.\"\n                    upload_date_current_flow = dateutil.parser.parse(flow1.upload_date)\n                    upload_date_parent_flow = dateutil.parser.parse(\n                        ignore_parameter_values_on_older_children,\n                    )\n                    if upload_date_current_flow &lt; upload_date_parent_flow:\n                        continue\n\n                if ignore_parameter_values:\n                    # Continue needs to be done here as the first if\n                    # statement triggers in both special cases\n                    continue\n            elif (\n                key == \"custom_name\"\n                and ignore_custom_name_if_none\n                and (attr1 is None or attr2 is None)\n            ):\n                # If specified, we allow `custom_name` inequality if one flow's name is None.\n                # Helps with backwards compatibility as `custom_name` is now auto-generated, but\n                # before it used to be `None`.\n                continue\n            elif key == \"parameters_meta_info\":\n                # this value is a dictionary where each key is a parameter name, containing another\n                # dictionary with keys specifying the parameter's 'description' and 'data_type'\n                # checking parameter descriptions can be ignored since that might change\n                # data type check can also be ignored if one of them is not defined, i.e., None\n                params1 = set(flow1.parameters_meta_info)\n                params2 = set(flow2.parameters_meta_info)\n                if params1 != params2:\n                    raise ValueError(\n                        \"Parameter list in meta info for parameters differ \" \"in the two flows.\",\n                    )\n                # iterating over the parameter's meta info list\n                for param in params1:\n                    if (\n                        isinstance(flow1.parameters_meta_info[param], Dict)\n                        and isinstance(flow2.parameters_meta_info[param], Dict)\n                        and \"data_type\" in flow1.parameters_meta_info[param]\n                        and \"data_type\" in flow2.parameters_meta_info[param]\n                    ):\n                        value1 = flow1.parameters_meta_info[param][\"data_type\"]\n                        value2 = flow2.parameters_meta_info[param][\"data_type\"]\n                    else:\n                        value1 = flow1.parameters_meta_info[param]\n                        value2 = flow2.parameters_meta_info[param]\n                    if value1 is None or value2 is None:\n                        continue\n\n                    if value1 != value2:\n                        raise ValueError(\n                            f\"Flow {flow1.name}: data type for parameter {param} in {key} differ \"\n                            f\"as {value1}\\nvs\\n{value2}\",\n                        )\n                # the continue is to avoid the 'attr != attr2' check at end of function\n                continue\n\n            if attr1 != attr2:\n                raise ValueError(\n                    f\"Flow {flow1.name!s}: values for attribute '{key!s}' differ: \"\n                    f\"'{attr1!s}'\\nvs\\n'{attr2!s}'.\",\n                )\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.delete_flow","title":"<code>delete_flow(flow_id)</code>","text":"<p>Delete flow with id <code>flow_id</code> from the OpenML server.</p> <p>You can only delete flows which you uploaded and which which are not linked to runs.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>OpenML id of the flow</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def delete_flow(flow_id: int) -&gt; bool:\n    \"\"\"Delete flow with id `flow_id` from the OpenML server.\n\n    You can only delete flows which you uploaded and which\n    which are not linked to runs.\n\n    Parameters\n    ----------\n    flow_id : int\n        OpenML id of the flow\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"flow\", flow_id)\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.flow_exists","title":"<code>flow_exists(name, external_version)</code>","text":"<p>Retrieves the flow id.</p> <p>A flow is uniquely identified by name + external_version.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>string</code> <p>Name of the flow</p> required <code>external_version</code> <code>string</code> <p>Version information associated with flow.</p> required <p>Returns:</p> Name Type Description <code>flow_exist</code> <code>int or bool</code> <p>flow id iff exists, False otherwise</p> Notes <p>see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version</p> Source code in <code>openml/flows/functions.py</code> <pre><code>def flow_exists(name: str, external_version: str) -&gt; int | bool:\n    \"\"\"Retrieves the flow id.\n\n    A flow is uniquely identified by name + external_version.\n\n    Parameters\n    ----------\n    name : string\n        Name of the flow\n    external_version : string\n        Version information associated with flow.\n\n    Returns\n    -------\n    flow_exist : int or bool\n        flow id iff exists, False otherwise\n\n    Notes\n    -----\n    see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version\n    \"\"\"\n    if not (isinstance(name, str) and len(name) &gt; 0):\n        raise ValueError(\"Argument 'name' should be a non-empty string\")\n    if not (isinstance(name, str) and len(external_version) &gt; 0):\n        raise ValueError(\"Argument 'version' should be a non-empty string\")\n\n    xml_response = openml._api_calls._perform_api_call(\n        \"flow/exists\",\n        \"post\",\n        data={\"name\": name, \"external_version\": external_version},\n    )\n\n    result_dict = xmltodict.parse(xml_response)\n    flow_id = int(result_dict[\"oml:flow_exists\"][\"oml:id\"])\n    return flow_id if flow_id &gt; 0 else False\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.get_flow","title":"<code>get_flow(flow_id, reinstantiate=False, strict_version=True)</code>","text":"<p>Download the OpenML flow for a given flow ID.</p> <p>Parameters:</p> Name Type Description Default <code>flow_id</code> <code>int</code> <p>The OpenML flow id.</p> required <code>reinstantiate</code> <code>bool</code> <p>Whether to reinstantiate the flow to a model instance.</p> <code>False</code> <code>strict_version</code> <code>bool</code> <p>Whether to fail if version requirements are not fulfilled.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>flow</code> <code>OpenMLFlow</code> <p>the flow</p> Source code in <code>openml/flows/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_flow(flow_id: int, reinstantiate: bool = False, strict_version: bool = True) -&gt; OpenMLFlow:  # noqa: FBT001, FBT002\n    \"\"\"Download the OpenML flow for a given flow ID.\n\n    Parameters\n    ----------\n    flow_id : int\n        The OpenML flow id.\n\n    reinstantiate: bool\n        Whether to reinstantiate the flow to a model instance.\n\n    strict_version : bool, default=True\n        Whether to fail if version requirements are not fulfilled.\n\n    Returns\n    -------\n    flow : OpenMLFlow\n        the flow\n    \"\"\"\n    flow_id = int(flow_id)\n    flow = _get_flow_description(flow_id)\n\n    if reinstantiate:\n        flow.model = flow.extension.flow_to_model(flow, strict_version=strict_version)\n        if not strict_version:\n            # check if we need to return a new flow b/c of version mismatch\n            new_flow = flow.extension.model_to_flow(flow.model)\n            if new_flow.dependencies != flow.dependencies:\n                return new_flow\n    return flow\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.get_flow_id","title":"<code>get_flow_id(model=None, name=None, exact_version=True)</code>","text":"<p>Retrieves the flow id for a model or a flow name.</p> <p>Provide either a model or a name to this function. Depending on the input, it does</p> <ul> <li><code>model</code> and <code>exact_version == True</code>: This helper function first queries for the necessary   extension. Second, it uses that extension to convert the model into a flow. Third, it   executes <code>flow_exists</code> to potentially obtain the flow id the flow is published to the   server.</li> <li><code>model</code> and <code>exact_version == False</code>: This helper function first queries for the   necessary extension. Second, it uses that extension to convert the model into a flow. Third   it calls <code>list_flows</code> and filters the returned values based on the flow name.</li> <li><code>name</code>: Ignores <code>exact_version</code> and calls <code>list_flows</code>, then filters the returned   values based on the flow name.</li> </ul> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>object</code> <p>Any model. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>name</code> <code>str</code> <p>Name of the flow. Must provide either <code>model</code> or <code>name</code>.</p> <code>None</code> <code>exact_version</code> <code>bool</code> <p>Whether to return the flow id of the exact version or all flow ids where the name of the flow matches. This is only taken into account for a model where a version number is available (requires <code>model</code> to be set).</p> <code>True</code> <p>Returns:</p> Type Description <code>(int or bool, List)</code> <p>flow id iff exists, <code>False</code> otherwise, List if <code>exact_version is False</code></p> Source code in <code>openml/flows/functions.py</code> <pre><code>def get_flow_id(\n    model: Any | None = None,\n    name: str | None = None,\n    exact_version: bool = True,  # noqa: FBT001, FBT002\n) -&gt; int | bool | list[int]:\n    \"\"\"Retrieves the flow id for a model or a flow name.\n\n    Provide either a model or a name to this function. Depending on the input, it does\n\n    * ``model`` and ``exact_version == True``: This helper function first queries for the necessary\n      extension. Second, it uses that extension to convert the model into a flow. Third, it\n      executes ``flow_exists`` to potentially obtain the flow id the flow is published to the\n      server.\n    * ``model`` and ``exact_version == False``: This helper function first queries for the\n      necessary extension. Second, it uses that extension to convert the model into a flow. Third\n      it calls ``list_flows`` and filters the returned values based on the flow name.\n    * ``name``: Ignores ``exact_version`` and calls ``list_flows``, then filters the returned\n      values based on the flow name.\n\n    Parameters\n    ----------\n    model : object\n        Any model. Must provide either ``model`` or ``name``.\n    name : str\n        Name of the flow. Must provide either ``model`` or ``name``.\n    exact_version : bool\n        Whether to return the flow id of the exact version or all flow ids where the name\n        of the flow matches. This is only taken into account for a model where a version number\n        is available (requires ``model`` to be set).\n\n    Returns\n    -------\n    int or bool, List\n        flow id iff exists, ``False`` otherwise, List if ``exact_version is False``\n    \"\"\"\n    if model is not None and name is not None:\n        raise ValueError(\"Must provide either argument `model` or argument `name`, but not both.\")\n\n    if model is not None:\n        extension = openml.extensions.get_extension_by_model(model, raise_if_no_extension=True)\n        if extension is None:\n            # This should never happen and is only here to please mypy will be gone soon once the\n            # whole function is removed\n            raise TypeError(extension)\n        flow = extension.model_to_flow(model)\n        flow_name = flow.name\n        external_version = flow.external_version\n    elif name is not None:\n        flow_name = name\n        exact_version = False\n        external_version = None\n    else:\n        raise ValueError(\n            \"Need to provide either argument `model` or argument `name`, but both are `None`.\"\n        )\n\n    if exact_version:\n        if external_version is None:\n            raise ValueError(\"exact_version should be False if model is None!\")\n        return flow_exists(name=flow_name, external_version=external_version)\n\n    flows = list_flows(output_format=\"dataframe\")\n    assert isinstance(flows, pd.DataFrame)  # Make mypy happy\n    flows = flows.query(f'name == \"{flow_name}\"')\n    return flows[\"id\"].to_list()  # type: ignore[no-any-return]\n</code></pre>"},{"location":"reference/flows/functions/#openml.flows.functions.list_flows","title":"<code>list_flows(offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., output_format: Literal['dict'] = 'dict', **kwargs: Any) -&gt; dict\n</code></pre><pre><code>list_flows(offset: int | None = ..., size: int | None = ..., tag: str | None = ..., *, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre><pre><code>list_flows(offset: int | None, size: int | None, tag: str | None, output_format: Literal['dataframe'], **kwargs: Any) -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all flows which are on OpenML. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of flows to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of flows to return</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: uploader.</p> <code>{}</code> <p>Returns:</p> Name Type Description <code>flows</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     A mapping from flow_id to a dict giving a brief overview of the     respective flow.     Every flow is represented by a dictionary containing     the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> <li> <p>If output_format='dataframe'     Each row maps to a dataset     Each column contains the following information:</p> <ul> <li>flow id</li> <li>full name</li> <li>name</li> <li>version</li> <li>external version</li> <li>uploader</li> </ul> </li> </ul> Source code in <code>openml/flows/functions.py</code> <pre><code>def list_flows(\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all flows which are on OpenML.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of flows to skip, starting from the first\n    size : int, optional\n        the maximum number of flows to return\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: uploader.\n\n    Returns\n    -------\n    flows : dict of dicts, or dataframe\n        - If output_format='dict'\n            A mapping from flow_id to a dict giving a brief overview of the\n            respective flow.\n            Every flow is represented by a dictionary containing\n            the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n\n        - If output_format='dataframe'\n            Each row maps to a dataset\n            Each column contains the following information:\n            - flow id\n            - full name\n            - name\n            - version\n            - external version\n            - uploader\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(\n        list_output_format=output_format,\n        listing_call=_list_flows,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/runs/","title":"runs","text":""},{"location":"reference/runs/#openml.runs.OpenMLRun","title":"<code>OpenMLRun</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Run: result of running a model on an OpenML dataset.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The ID of the OpenML task associated with the run.</p> required <code>flow_id</code> <code>int | None</code> <p>The ID of the OpenML flow associated with the run.</p> required <code>dataset_id</code> <code>int | None</code> <p>The ID of the OpenML dataset used for the run.</p> required <code>setup_string</code> <code>str | None</code> <p>The setup string of the run.</p> <code>None</code> <code>output_files</code> <code>dict[str, int] | None</code> <p>Specifies where each related file can be found.</p> <code>None</code> <code>setup_id</code> <code>int | None</code> <p>An integer representing the ID of the setup used for the run.</p> <code>None</code> <code>tags</code> <code>list[str] | None</code> <p>Representing the tags associated with the run.</p> <code>None</code> <code>uploader</code> <code>int | None</code> <p>User ID of the uploader.</p> <code>None</code> <code>uploader_name</code> <code>str | None</code> <p>The name of the person who uploaded the run.</p> <code>None</code> <code>evaluations</code> <code>dict | None</code> <p>Representing the evaluations of the run.</p> <code>None</code> <code>fold_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each fold.</p> <code>None</code> <code>sample_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each sample.</p> <code>None</code> <code>data_content</code> <code>list[list] | None</code> <p>The predictions generated from executing this run.</p> <code>None</code> <code>trace</code> <code>OpenMLRunTrace | None</code> <p>The trace containing information on internal model evaluations of this run.</p> <code>None</code> <code>model</code> <code>object | None</code> <p>The untrained model that was evaluated in the run.</p> <code>None</code> <code>task_type</code> <code>str | None</code> <p>The type of the OpenML task associated with the run.</p> <code>None</code> <code>task_evaluation_measure</code> <code>str | None</code> <p>The evaluation measure used for the task.</p> <code>None</code> <code>flow_name</code> <code>str | None</code> <p>The name of the OpenML flow associated with the run.</p> <code>None</code> <code>parameter_settings</code> <code>list[dict[str, Any]] | None</code> <p>Representing the parameter settings used for the run.</p> <code>None</code> <code>predictions_url</code> <code>str | None</code> <p>The URL of the predictions file.</p> <code>None</code> <code>task</code> <code>OpenMLTask | None</code> <p>An instance of the OpenMLTask class, representing the OpenML task associated with the run.</p> <code>None</code> <code>flow</code> <code>OpenMLFlow | None</code> <p>An instance of the OpenMLFlow class, representing the OpenML flow associated with the run.</p> <code>None</code> <code>run_id</code> <code>int | None</code> <p>The ID of the run.</p> <code>None</code> <code>description_text</code> <code>str | None</code> <p>Description text to add to the predictions file. If left None, is set to the time the arff file is generated.</p> <code>None</code> <code>run_details</code> <code>str | None</code> <p>Description of the run stored in the run meta-data.</p> <code>None</code> Source code in <code>openml/runs/run.py</code> <pre><code>class OpenMLRun(OpenMLBase):\n    \"\"\"OpenML Run: result of running a model on an OpenML dataset.\n\n    Parameters\n    ----------\n    task_id: int\n        The ID of the OpenML task associated with the run.\n    flow_id: int\n        The ID of the OpenML flow associated with the run.\n    dataset_id: int\n        The ID of the OpenML dataset used for the run.\n    setup_string: str\n        The setup string of the run.\n    output_files: Dict[str, int]\n        Specifies where each related file can be found.\n    setup_id: int\n        An integer representing the ID of the setup used for the run.\n    tags: List[str]\n        Representing the tags associated with the run.\n    uploader: int\n        User ID of the uploader.\n    uploader_name: str\n        The name of the person who uploaded the run.\n    evaluations: Dict\n        Representing the evaluations of the run.\n    fold_evaluations: Dict\n        The evaluations of the run for each fold.\n    sample_evaluations: Dict\n        The evaluations of the run for each sample.\n    data_content: List[List]\n        The predictions generated from executing this run.\n    trace: OpenMLRunTrace\n        The trace containing information on internal model evaluations of this run.\n    model: object\n        The untrained model that was evaluated in the run.\n    task_type: str\n        The type of the OpenML task associated with the run.\n    task_evaluation_measure: str\n        The evaluation measure used for the task.\n    flow_name: str\n        The name of the OpenML flow associated with the run.\n    parameter_settings: list[OrderedDict]\n        Representing the parameter settings used for the run.\n    predictions_url: str\n        The URL of the predictions file.\n    task: OpenMLTask\n        An instance of the OpenMLTask class, representing the OpenML task associated\n        with the run.\n    flow: OpenMLFlow\n        An instance of the OpenMLFlow class, representing the OpenML flow associated\n        with the run.\n    run_id: int\n        The ID of the run.\n    description_text: str, optional\n        Description text to add to the predictions file. If left None, is set to the\n        time the arff file is generated.\n    run_details: str, optional (default=None)\n        Description of the run stored in the run meta-data.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int,\n        flow_id: int | None,\n        dataset_id: int | None,\n        setup_string: str | None = None,\n        output_files: dict[str, int] | None = None,\n        setup_id: int | None = None,\n        tags: list[str] | None = None,\n        uploader: int | None = None,\n        uploader_name: str | None = None,\n        evaluations: dict | None = None,\n        fold_evaluations: dict | None = None,\n        sample_evaluations: dict | None = None,\n        data_content: list[list] | None = None,\n        trace: OpenMLRunTrace | None = None,\n        model: object | None = None,\n        task_type: str | None = None,\n        task_evaluation_measure: str | None = None,\n        flow_name: str | None = None,\n        parameter_settings: list[dict[str, Any]] | None = None,\n        predictions_url: str | None = None,\n        task: OpenMLTask | None = None,\n        flow: OpenMLFlow | None = None,\n        run_id: int | None = None,\n        description_text: str | None = None,\n        run_details: str | None = None,\n    ):\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.task_id = task_id\n        self.task_type = task_type\n        self.task_evaluation_measure = task_evaluation_measure\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.setup_id = setup_id\n        self.setup_string = setup_string\n        self.parameter_settings = parameter_settings\n        self.dataset_id = dataset_id\n        self.evaluations = evaluations\n        self.fold_evaluations = fold_evaluations\n        self.sample_evaluations = sample_evaluations\n        self.data_content = data_content\n        self.output_files = output_files\n        self.trace = trace\n        self.error_message = None\n        self.task = task\n        self.flow = flow\n        self.run_id = run_id\n        self.model = model\n        self.tags = tags\n        self.predictions_url = predictions_url\n        self.description_text = description_text\n        self.run_details = run_details\n        self._predictions = None\n\n    @property\n    def predictions(self) -&gt; pd.DataFrame:\n        \"\"\"Return a DataFrame with predictions for this run\"\"\"\n        if self._predictions is None:\n            if self.data_content:\n                arff_dict = self._generate_arff_dict()\n            elif self.predictions_url:\n                arff_text = openml._api_calls._download_text_file(self.predictions_url)\n                arff_dict = arff.loads(arff_text)\n            else:\n                raise RuntimeError(\"Run has no predictions.\")\n            self._predictions = pd.DataFrame(\n                arff_dict[\"data\"],\n                columns=[name for name, _ in arff_dict[\"attributes\"]],\n            )\n        return self._predictions\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the run, None if not uploaded to the server yet.\"\"\"\n        return self.run_id\n\n    def _evaluation_summary(self, metric: str) -&gt; str:\n        \"\"\"Summarizes the evaluation of a metric over all folds.\n\n        The fold scores for the metric must exist already. During run creation,\n        by default, the MAE for OpenMLRegressionTask and the accuracy for\n        OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed.\n\n        If repetition exist, we take the mean over all repetitions.\n\n        Parameters\n        ----------\n        metric: str\n            Name of an evaluation metric that was used to compute fold scores.\n\n        Returns\n        -------\n        metric_summary: str\n            A formatted string that displays the metric's evaluation summary.\n            The summary consists of the mean and std.\n        \"\"\"\n        if self.fold_evaluations is None:\n            raise ValueError(\"No fold evaluations available.\")\n        fold_score_lists = self.fold_evaluations[metric].values()\n\n        # Get the mean and std over all repetitions\n        rep_means = [np.mean(list(x.values())) for x in fold_score_lists]\n        rep_stds = [np.std(list(x.values())) for x in fold_score_lists]\n\n        return f\"{np.mean(rep_means):.4f} +- {np.mean(rep_stds):.4f}\"\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Set up fields\n        fields = {\n            \"Uploader Name\": self.uploader_name,\n            \"Metric\": self.task_evaluation_measure,\n            \"Run ID\": self.run_id,\n            \"Task ID\": self.task_id,\n            \"Task Type\": self.task_type,\n            \"Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"Flow Name\": self.flow_name,\n            \"Flow URL\": (\n                openml.flows.OpenMLFlow.url_for_id(self.flow_id)\n                if self.flow_id is not None\n                else None\n            ),\n            \"Setup ID\": self.setup_id,\n            \"Setup String\": self.setup_string,\n            \"Dataset ID\": self.dataset_id,\n            \"Dataset URL\": (\n                openml.datasets.OpenMLDataset.url_for_id(self.dataset_id)\n                if self.dataset_id is not None\n                else None\n            ),\n        }\n\n        # determines the order of the initial fields in which the information will be printed\n        order = [\"Uploader Name\", \"Uploader Profile\", \"Metric\", \"Result\"]\n\n        if self.uploader is not None:\n            fields[\"Uploader Profile\"] = f\"{openml.config.get_server_base_url()}/u/{self.uploader}\"\n        if self.run_id is not None:\n            fields[\"Run URL\"] = self.openml_url\n        if self.evaluations is not None and self.task_evaluation_measure in self.evaluations:\n            fields[\"Result\"] = self.evaluations[self.task_evaluation_measure]\n        elif self.fold_evaluations is not None:\n            # -- Add locally computed summary values if possible\n            if \"predictive_accuracy\" in self.fold_evaluations:\n                # OpenMLClassificationTask; OpenMLLearningCurveTask\n                result_field = \"Local Result - Accuracy (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"predictive_accuracy\")\n                order.append(result_field)\n            elif \"mean_absolute_error\" in self.fold_evaluations:\n                # OpenMLRegressionTask\n                result_field = \"Local Result - MAE (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"mean_absolute_error\")\n                order.append(result_field)\n\n            if \"usercpu_time_millis\" in self.fold_evaluations:\n                # Runtime should be available for most tasks types\n                rt_field = \"Local Runtime - ms (+- STD)\"\n                fields[rt_field] = self._evaluation_summary(\"usercpu_time_millis\")\n                order.append(rt_field)\n\n        # determines the remaining order\n        order += [\n            \"Run ID\",\n            \"Run URL\",\n            \"Task ID\",\n            \"Task Type\",\n            \"Task URL\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Setup ID\",\n            \"Setup String\",\n            \"Dataset ID\",\n            \"Dataset URL\",\n        ]\n        return [\n            (key, \"None\" if fields[key] is None else fields[key])  # type: ignore\n            for key in order\n            if key in fields\n        ]\n\n    @classmethod\n    def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n        \"\"\"\n        The inverse of the to_filesystem method. Instantiates an OpenMLRun\n        object based on files stored on the file system.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            are stored\n\n        expect_model : bool\n            if True, it requires the model pickle to be present, and an error\n            will be thrown if not. Otherwise, the model might or might not\n            be present.\n\n        Returns\n        -------\n        run : OpenMLRun\n            the re-instantiated run object\n        \"\"\"\n        # Avoiding cyclic imports\n        import openml.runs.functions\n\n        directory = Path(directory)\n        if not directory.is_dir():\n            raise ValueError(\"Could not find folder\")\n\n        description_path = directory / \"description.xml\"\n        predictions_path = directory / \"predictions.arff\"\n        trace_path = directory / \"trace.arff\"\n        model_path = directory / \"model.pkl\"\n\n        if not description_path.is_file():\n            raise ValueError(\"Could not find description.xml\")\n        if not predictions_path.is_file():\n            raise ValueError(\"Could not find predictions.arff\")\n        if (not model_path.is_file()) and expect_model:\n            raise ValueError(\"Could not find model.pkl\")\n\n        with description_path.open() as fht:\n            xml_string = fht.read()\n        run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n        if run.flow_id is None:\n            flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n            run.flow = flow\n            run.flow_name = flow.name\n\n        with predictions_path.open() as fht:\n            predictions = arff.load(fht)\n            run.data_content = predictions[\"data\"]\n\n        if model_path.is_file():\n            # note that it will load the model if the file exists, even if\n            # expect_model is False\n            with model_path.open(\"rb\") as fhb:\n                run.model = pickle.load(fhb)  # noqa: S301\n\n        if trace_path.is_file():\n            run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n        return run\n\n    def to_filesystem(\n        self,\n        directory: str | Path,\n        store_model: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        \"\"\"\n        The inverse of the from_filesystem method. Serializes a run\n        on the filesystem, to be uploaded later.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            will be stored. Should be empty\n\n        store_model : bool, optional (default=True)\n            if True, a model will be pickled as well. As this is the most\n            storage expensive part, it is often desirable to not store the\n            model.\n        \"\"\"\n        if self.data_content is None or self.model is None:\n            raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n        directory = Path(directory)\n        directory.mkdir(exist_ok=True, parents=True)\n\n        if any(directory.iterdir()):\n            raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n        run_xml = self._to_xml()\n        predictions_arff = arff.dumps(self._generate_arff_dict())\n\n        # It seems like typing does not allow to define the same variable multiple times\n        with (directory / \"description.xml\").open(\"w\") as fh:\n            fh.write(run_xml)\n        with (directory / \"predictions.arff\").open(\"w\") as fh:\n            fh.write(predictions_arff)\n        if store_model:\n            with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n                pickle.dump(self.model, fh_b)\n\n        if self.flow_id is None and self.flow is not None:\n            self.flow.to_filesystem(directory)\n\n        if self.trace is not None:\n            self.trace._to_filesystem(directory)\n\n    def _generate_arff_dict(self) -&gt; OrderedDict[str, Any]:\n        \"\"\"Generates the arff dictionary for uploading predictions to the\n        server.\n\n        Assumes that the run has been executed.\n\n        The order of the attributes follows the order defined by the Client API for R.\n\n        Returns\n        -------\n        arf_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains predictions and information about the run environment.\n        \"\"\"\n        if self.data_content is None:\n            raise ValueError(\"Run has not been executed.\")\n        if self.flow is None:\n            assert self.flow_id is not None, \"Run has no associated flow id!\"\n            self.flow = get_flow(self.flow_id)\n\n        if self.description_text is None:\n            self.description_text = time.strftime(\"%c\")\n        task = get_task(self.task_id)\n\n        arff_dict = OrderedDict()  # type: 'OrderedDict[str, Any]'\n        arff_dict[\"data\"] = self.data_content\n        arff_dict[\"description\"] = self.description_text\n        arff_dict[\"relation\"] = f\"openml_task_{task.task_id}_predictions\"\n\n        if isinstance(task, OpenMLLearningCurveTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"]\n                    + [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                    + [\n                        (\"confidence.\" + class_labels[i], \"NUMERIC\")\n                        for i in range(len(class_labels))\n                    ]\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLClassificationTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),  # Legacy\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                prediction_confidences = [\n                    (\"confidence.\" + class_labels[i], \"NUMERIC\") for i in range(len(class_labels))\n                ]\n                prediction_and_true = [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"] + prediction_and_true + prediction_confidences\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"prediction\", \"NUMERIC\"),\n                (\"truth\", \"NUMERIC\"),\n            ]\n\n        elif isinstance(task, OpenMLClusteringTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"cluster\", \"NUMERIC\"),\n            ]\n\n        else:\n            raise NotImplementedError(\"Task type %s is not yet supported.\" % str(task.task_type))\n\n        return arff_dict\n\n    def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n        \"\"\"Calculates metric scores based on predicted values. Assumes the\n        run has been executed locally (and contains run_data). Furthermore,\n        it assumes that the 'correct' or 'truth' attribute is specified in\n        the arff (which is an optional field, but always the case for\n        openml-python runs)\n\n        Parameters\n        ----------\n        sklearn_fn : function\n            a function pointer to a sklearn function that\n            accepts ``y_true``, ``y_pred`` and ``**kwargs``\n        kwargs : dict\n            kwargs for the function\n\n        Returns\n        -------\n        scores : ndarray of scores of length num_folds * num_repeats\n            metric results\n        \"\"\"\n        kwargs = kwargs if kwargs else {}\n        if self.data_content is not None and self.task_id is not None:\n            predictions_arff = self._generate_arff_dict()\n        elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n            predictions_file_url = openml._api_calls._file_id_to_url(\n                self.output_files[\"predictions\"],\n                \"predictions.arff\",\n            )\n            response = openml._api_calls._download_text_file(predictions_file_url)\n            predictions_arff = arff.loads(response)\n            # TODO: make this a stream reader\n        else:\n            raise ValueError(\n                \"Run should have been locally executed or \" \"contain outputfile reference.\",\n            )\n\n        # Need to know more about the task to compute scores correctly\n        task = get_task(self.task_id)\n\n        attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n        if (\n            task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n            and \"correct\" not in attribute_names\n        ):\n            raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n        if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n            raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n        if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n            raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n        def _attribute_list_to_dict(attribute_list):  # type: ignore\n            # convenience function: Creates a mapping to map from the name of\n            # attributes present in the arff prediction file to their index.\n            # This is necessary because the number of classes can be different\n            # for different tasks.\n            res = OrderedDict()\n            for idx in range(len(attribute_list)):\n                res[attribute_list[idx][0]] = idx\n            return res\n\n        attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n        repeat_idx = attribute_dict[\"repeat\"]\n        fold_idx = attribute_dict[\"fold\"]\n        predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n        if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n            correct_idx = attribute_dict[\"correct\"]\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            correct_idx = attribute_dict[\"truth\"]\n        has_samples = False\n        if \"sample\" in attribute_dict:\n            sample_idx = attribute_dict[\"sample\"]\n            has_samples = True\n\n        if (\n            predictions_arff[\"attributes\"][predicted_idx][1]\n            != predictions_arff[\"attributes\"][correct_idx][1]\n        ):\n            pred = predictions_arff[\"attributes\"][predicted_idx][1]\n            corr = predictions_arff[\"attributes\"][correct_idx][1]\n            raise ValueError(\n                \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n            )\n\n        # TODO: these could be cached\n        values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n        values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n        for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n            rep = line[repeat_idx]\n            fold = line[fold_idx]\n            samp = line[sample_idx] if has_samples else 0\n\n            if task.task_type_id in [\n                TaskType.SUPERVISED_CLASSIFICATION,\n                TaskType.LEARNING_CURVE,\n            ]:\n                prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                    line[predicted_idx],\n                )\n                correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n            elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n                prediction = line[predicted_idx]\n                correct = line[correct_idx]\n            if rep not in values_predict:\n                values_predict[rep] = OrderedDict()\n                values_correct[rep] = OrderedDict()\n            if fold not in values_predict[rep]:\n                values_predict[rep][fold] = OrderedDict()\n                values_correct[rep][fold] = OrderedDict()\n            if samp not in values_predict[rep][fold]:\n                values_predict[rep][fold][samp] = []\n                values_correct[rep][fold][samp] = []\n\n            values_predict[rep][fold][samp].append(prediction)\n            values_correct[rep][fold][samp].append(correct)\n\n        scores = []\n        for rep in values_predict:\n            for fold in values_predict[rep]:\n                last_sample = len(values_predict[rep][fold]) - 1\n                y_pred = values_predict[rep][fold][last_sample]\n                y_true = values_correct[rep][fold][last_sample]\n                scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n        return np.array(scores)\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.run_id = int(xml_response[\"oml:upload_run\"][\"oml:run_id\"])\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Get file_elements to upload to the server.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        if self.parameter_settings is None and self.model is None:\n            raise PyOpenMLError(\n                \"OpenMLRun must contain a model or be initialized with parameter_settings.\",\n            )\n        if self.flow_id is None:\n            if self.flow is None:\n                raise PyOpenMLError(\n                    \"OpenMLRun object does not contain a flow id or reference to OpenMLFlow \"\n                    \"(these should have been added while executing the task). \",\n                )\n\n            # publish the linked Flow before publishing the run.\n            self.flow.publish()\n            self.flow_id = self.flow.flow_id\n\n        if self.parameter_settings is None:\n            if self.flow is None:\n                assert self.flow_id is not None  # for mypy\n                self.flow = openml.flows.get_flow(self.flow_id)\n            self.parameter_settings = self.flow.extension.obtain_parameter_values(\n                self.flow,\n                self.model,\n            )\n\n        file_elements = {\"description\": (\"description.xml\", self._to_xml())}\n\n        if self.error_message is None:\n            predictions = arff.dumps(self._generate_arff_dict())\n            file_elements[\"predictions\"] = (\"predictions.arff\", predictions)\n\n        if self.trace is not None:\n            trace_arff = arff.dumps(self.trace.trace_to_arff())\n            file_elements[\"trace\"] = (\"trace.arff\", trace_arff)\n        return file_elements\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: PLR0912, C901\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        description = OrderedDict()  # type: 'OrderedDict'\n        description[\"oml:run\"] = OrderedDict()\n        description[\"oml:run\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n        description[\"oml:run\"][\"oml:task_id\"] = self.task_id\n        description[\"oml:run\"][\"oml:flow_id\"] = self.flow_id\n        if self.setup_string is not None:\n            description[\"oml:run\"][\"oml:setup_string\"] = self.setup_string\n        if self.error_message is not None:\n            description[\"oml:run\"][\"oml:error_message\"] = self.error_message\n        if self.run_details is not None:\n            description[\"oml:run\"][\"oml:run_details\"] = self.run_details\n        description[\"oml:run\"][\"oml:parameter_setting\"] = self.parameter_settings\n        if self.tags is not None:\n            description[\"oml:run\"][\"oml:tag\"] = self.tags\n        if (self.fold_evaluations is not None and len(self.fold_evaluations) &gt; 0) or (\n            self.sample_evaluations is not None and len(self.sample_evaluations) &gt; 0\n        ):\n            description[\"oml:run\"][\"oml:output_data\"] = OrderedDict()\n            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"] = []\n        if self.fold_evaluations is not None:\n            for measure in self.fold_evaluations:\n                for repeat in self.fold_evaluations[measure]:\n                    for fold, value in self.fold_evaluations[measure][repeat].items():\n                        current = OrderedDict(\n                            [\n                                (\"@repeat\", str(repeat)),\n                                (\"@fold\", str(fold)),\n                                (\"oml:name\", measure),\n                                (\"oml:value\", str(value)),\n                            ],\n                        )\n                        description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(current)\n        if self.sample_evaluations is not None:\n            for measure in self.sample_evaluations:\n                for repeat in self.sample_evaluations[measure]:\n                    for fold in self.sample_evaluations[measure][repeat]:\n                        for sample, value in self.sample_evaluations[measure][repeat][fold].items():\n                            current = OrderedDict(\n                                [\n                                    (\"@repeat\", str(repeat)),\n                                    (\"@fold\", str(fold)),\n                                    (\"@sample\", str(sample)),\n                                    (\"oml:name\", measure),\n                                    (\"oml:value\", str(value)),\n                                ],\n                            )\n                            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(\n                                current,\n                            )\n        return description\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRun.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the run, None if not uploaded to the server yet.</p>"},{"location":"reference/runs/#openml.runs.OpenMLRun.predictions","title":"<code>predictions: pd.DataFrame</code>  <code>property</code>","text":"<p>Return a DataFrame with predictions for this run</p>"},{"location":"reference/runs/#openml.runs.OpenMLRun.from_filesystem","title":"<code>from_filesystem(directory, expect_model=True)</code>  <code>classmethod</code>","text":"<p>The inverse of the to_filesystem method. Instantiates an OpenMLRun object based on files stored on the file system.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results are stored</p> required <code>expect_model</code> <code>bool</code> <p>if True, it requires the model pickle to be present, and an error will be thrown if not. Otherwise, the model might or might not be present.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>the re-instantiated run object</p> Source code in <code>openml/runs/run.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n    \"\"\"\n    The inverse of the to_filesystem method. Instantiates an OpenMLRun\n    object based on files stored on the file system.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        are stored\n\n    expect_model : bool\n        if True, it requires the model pickle to be present, and an error\n        will be thrown if not. Otherwise, the model might or might not\n        be present.\n\n    Returns\n    -------\n    run : OpenMLRun\n        the re-instantiated run object\n    \"\"\"\n    # Avoiding cyclic imports\n    import openml.runs.functions\n\n    directory = Path(directory)\n    if not directory.is_dir():\n        raise ValueError(\"Could not find folder\")\n\n    description_path = directory / \"description.xml\"\n    predictions_path = directory / \"predictions.arff\"\n    trace_path = directory / \"trace.arff\"\n    model_path = directory / \"model.pkl\"\n\n    if not description_path.is_file():\n        raise ValueError(\"Could not find description.xml\")\n    if not predictions_path.is_file():\n        raise ValueError(\"Could not find predictions.arff\")\n    if (not model_path.is_file()) and expect_model:\n        raise ValueError(\"Could not find model.pkl\")\n\n    with description_path.open() as fht:\n        xml_string = fht.read()\n    run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n    if run.flow_id is None:\n        flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n        run.flow = flow\n        run.flow_name = flow.name\n\n    with predictions_path.open() as fht:\n        predictions = arff.load(fht)\n        run.data_content = predictions[\"data\"]\n\n    if model_path.is_file():\n        # note that it will load the model if the file exists, even if\n        # expect_model is False\n        with model_path.open(\"rb\") as fhb:\n            run.model = pickle.load(fhb)  # noqa: S301\n\n    if trace_path.is_file():\n        run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n    return run\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRun.get_metric_fn","title":"<code>get_metric_fn(sklearn_fn, kwargs=None)</code>","text":"<p>Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' or 'truth' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs)</p> <p>Parameters:</p> Name Type Description Default <code>sklearn_fn</code> <code>function</code> <p>a function pointer to a sklearn function that accepts <code>y_true</code>, <code>y_pred</code> and <code>**kwargs</code></p> required <code>kwargs</code> <code>dict</code> <p>kwargs for the function</p> <code>None</code> <p>Returns:</p> Name Type Description <code>scores</code> <code>ndarray of scores of length num_folds * num_repeats</code> <p>metric results</p> Source code in <code>openml/runs/run.py</code> <pre><code>def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n    \"\"\"Calculates metric scores based on predicted values. Assumes the\n    run has been executed locally (and contains run_data). Furthermore,\n    it assumes that the 'correct' or 'truth' attribute is specified in\n    the arff (which is an optional field, but always the case for\n    openml-python runs)\n\n    Parameters\n    ----------\n    sklearn_fn : function\n        a function pointer to a sklearn function that\n        accepts ``y_true``, ``y_pred`` and ``**kwargs``\n    kwargs : dict\n        kwargs for the function\n\n    Returns\n    -------\n    scores : ndarray of scores of length num_folds * num_repeats\n        metric results\n    \"\"\"\n    kwargs = kwargs if kwargs else {}\n    if self.data_content is not None and self.task_id is not None:\n        predictions_arff = self._generate_arff_dict()\n    elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n        predictions_file_url = openml._api_calls._file_id_to_url(\n            self.output_files[\"predictions\"],\n            \"predictions.arff\",\n        )\n        response = openml._api_calls._download_text_file(predictions_file_url)\n        predictions_arff = arff.loads(response)\n        # TODO: make this a stream reader\n    else:\n        raise ValueError(\n            \"Run should have been locally executed or \" \"contain outputfile reference.\",\n        )\n\n    # Need to know more about the task to compute scores correctly\n    task = get_task(self.task_id)\n\n    attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n    if (\n        task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n        and \"correct\" not in attribute_names\n    ):\n        raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n    if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n        raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n    if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n        raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n    def _attribute_list_to_dict(attribute_list):  # type: ignore\n        # convenience function: Creates a mapping to map from the name of\n        # attributes present in the arff prediction file to their index.\n        # This is necessary because the number of classes can be different\n        # for different tasks.\n        res = OrderedDict()\n        for idx in range(len(attribute_list)):\n            res[attribute_list[idx][0]] = idx\n        return res\n\n    attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n    repeat_idx = attribute_dict[\"repeat\"]\n    fold_idx = attribute_dict[\"fold\"]\n    predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n    if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n        correct_idx = attribute_dict[\"correct\"]\n    elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n        correct_idx = attribute_dict[\"truth\"]\n    has_samples = False\n    if \"sample\" in attribute_dict:\n        sample_idx = attribute_dict[\"sample\"]\n        has_samples = True\n\n    if (\n        predictions_arff[\"attributes\"][predicted_idx][1]\n        != predictions_arff[\"attributes\"][correct_idx][1]\n    ):\n        pred = predictions_arff[\"attributes\"][predicted_idx][1]\n        corr = predictions_arff[\"attributes\"][correct_idx][1]\n        raise ValueError(\n            \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n        )\n\n    # TODO: these could be cached\n    values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n    values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n    for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n        rep = line[repeat_idx]\n        fold = line[fold_idx]\n        samp = line[sample_idx] if has_samples else 0\n\n        if task.task_type_id in [\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.LEARNING_CURVE,\n        ]:\n            prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                line[predicted_idx],\n            )\n            correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            prediction = line[predicted_idx]\n            correct = line[correct_idx]\n        if rep not in values_predict:\n            values_predict[rep] = OrderedDict()\n            values_correct[rep] = OrderedDict()\n        if fold not in values_predict[rep]:\n            values_predict[rep][fold] = OrderedDict()\n            values_correct[rep][fold] = OrderedDict()\n        if samp not in values_predict[rep][fold]:\n            values_predict[rep][fold][samp] = []\n            values_correct[rep][fold][samp] = []\n\n        values_predict[rep][fold][samp].append(prediction)\n        values_correct[rep][fold][samp].append(correct)\n\n    scores = []\n    for rep in values_predict:\n        for fold in values_predict[rep]:\n            last_sample = len(values_predict[rep][fold]) - 1\n            y_pred = values_predict[rep][fold][last_sample]\n            y_true = values_correct[rep][fold][last_sample]\n            scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n    return np.array(scores)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRun.to_filesystem","title":"<code>to_filesystem(directory, store_model=True)</code>","text":"<p>The inverse of the from_filesystem method. Serializes a run on the filesystem, to be uploaded later.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results will be stored. Should be empty</p> required <code>store_model</code> <code>(bool, optional(default=True))</code> <p>if True, a model will be pickled as well. As this is the most storage expensive part, it is often desirable to not store the model.</p> <code>True</code> Source code in <code>openml/runs/run.py</code> <pre><code>def to_filesystem(\n    self,\n    directory: str | Path,\n    store_model: bool = True,  # noqa: FBT001, FBT002\n) -&gt; None:\n    \"\"\"\n    The inverse of the from_filesystem method. Serializes a run\n    on the filesystem, to be uploaded later.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        will be stored. Should be empty\n\n    store_model : bool, optional (default=True)\n        if True, a model will be pickled as well. As this is the most\n        storage expensive part, it is often desirable to not store the\n        model.\n    \"\"\"\n    if self.data_content is None or self.model is None:\n        raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n    directory = Path(directory)\n    directory.mkdir(exist_ok=True, parents=True)\n\n    if any(directory.iterdir()):\n        raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n    run_xml = self._to_xml()\n    predictions_arff = arff.dumps(self._generate_arff_dict())\n\n    # It seems like typing does not allow to define the same variable multiple times\n    with (directory / \"description.xml\").open(\"w\") as fh:\n        fh.write(run_xml)\n    with (directory / \"predictions.arff\").open(\"w\") as fh:\n        fh.write(predictions_arff)\n    if store_model:\n        with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n            pickle.dump(self.model, fh_b)\n\n    if self.flow_id is None and self.flow is not None:\n        self.flow.to_filesystem(directory)\n\n    if self.trace is not None:\n        self.trace._to_filesystem(directory)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace","title":"<code>OpenMLRunTrace</code>","text":"<p>OpenML Run Trace: parsed output from Run Trace call</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML run id.</p> required <code>trace_iterations</code> <code>dict</code> <p>Mapping from key <code>(repeat, fold, iteration)</code> to an object of OpenMLTraceIteration.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>class OpenMLRunTrace:\n    \"\"\"OpenML Run Trace: parsed output from Run Trace call\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML run id.\n\n    trace_iterations : dict\n        Mapping from key ``(repeat, fold, iteration)`` to an object of\n        OpenMLTraceIteration.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        run_id: int | None,\n        trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n    ):\n        \"\"\"Object to hold the trace content of a run.\n\n        Parameters\n        ----------\n        run_id : int\n            Id for which the trace content is to be stored.\n        trace_iterations : List[List]\n            The trace content obtained by running a flow on a task.\n        \"\"\"\n        self.run_id = run_id\n        self.trace_iterations = trace_iterations\n\n    def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n        \"\"\"\n        Returns the trace iteration that was marked as selected. In\n        case multiple are marked as selected (should not happen) the\n        first of these is returned\n\n        Parameters\n        ----------\n        fold: int\n\n        repeat: int\n\n        Returns\n        -------\n        int\n            The trace iteration from the given fold and repeat that was\n            selected as the best iteration by the search procedure\n        \"\"\"\n        for r, f, i in self.trace_iterations:\n            if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n                return i\n        raise ValueError(\n            \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n        )\n\n    @classmethod\n    def generate(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Generates an OpenMLRunTrace.\n\n        Generates the trace object from the attributes and content extracted\n        while running the underlying flow.\n\n        Parameters\n        ----------\n        attributes : list\n            List of tuples describing the arff attributes.\n\n        content : list\n            List of lists containing information about the individual tuning\n            runs.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if content is None:\n            raise ValueError(\"Trace content not available.\")\n        if attributes is None:\n            raise ValueError(\"Trace attributes not available.\")\n        if len(content) == 0:\n            raise ValueError(\"Trace content is empty.\")\n        if len(attributes) != len(content[0]):\n            raise ValueError(\n                \"Trace_attributes and trace_content not compatible:\"\n                f\" {attributes} vs {content[0]}\",\n            )\n\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not allowed when constructing a \"\n            \"trace object from run results.\",\n        )\n\n    @classmethod\n    def _from_filesystem(cls, file_path: str | Path) -&gt; OpenMLRunTrace:\n        \"\"\"\n        Logic to deserialize the trace from the filesystem.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff is stored.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        file_path = Path(file_path)\n\n        if not file_path.exists():\n            raise ValueError(\"Trace file doesn't exist\")\n\n        with file_path.open(\"r\") as fp:\n            trace_arff = arff.load(fp)\n\n        for trace_idx in range(len(trace_arff[\"data\"])):\n            # iterate over first three entrees of a trace row\n            # (fold, repeat, trace_iteration) these should be int\n            for line_idx in range(3):\n                trace_arff[\"data\"][trace_idx][line_idx] = int(\n                    trace_arff[\"data\"][trace_idx][line_idx],\n                )\n\n        return cls.trace_from_arff(trace_arff)\n\n    def _to_filesystem(self, file_path: str | Path) -&gt; None:\n        \"\"\"Serialize the trace object to the filesystem.\n\n        Serialize the trace object as an arff.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff will be stored.\n        \"\"\"\n        trace_path = Path(file_path) / \"trace.arff\"\n\n        trace_arff = arff.dumps(self.trace_to_arff())\n        with trace_path.open(\"w\") as f:\n            f.write(trace_arff)\n\n    def trace_to_arff(self) -&gt; dict[str, Any]:\n        \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n        Uses the trace object to generate an arff dictionary representation.\n\n        Returns\n        -------\n        arff_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains information about the optimization trace.\n        \"\"\"\n        if self.trace_iterations is None:\n            raise ValueError(\"trace_iterations missing from the trace object\")\n\n        # attributes that will be in trace arff\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n        trace_attributes.extend(\n            [\n                (PREFIX + parameter, \"STRING\")\n                for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n            ],\n        )\n\n        arff_dict: dict[str, Any] = {}\n        data = []\n        for trace_iteration in self.trace_iterations.values():\n            tmp_list = []\n            for _attr, _ in trace_attributes:\n                if _attr.startswith(PREFIX):\n                    attr = _attr[len(PREFIX) :]\n                    value = trace_iteration.get_parameters()[attr]\n                else:\n                    attr = _attr\n                    value = getattr(trace_iteration, attr)\n\n                if attr == \"selected\":\n                    tmp_list.append(\"true\" if value else \"false\")\n                else:\n                    tmp_list.append(value)\n            data.append(tmp_list)\n\n        arff_dict[\"attributes\"] = trace_attributes\n        arff_dict[\"data\"] = data\n        # TODO allow to pass a trace description when running a flow\n        arff_dict[\"relation\"] = \"Trace\"\n        return arff_dict\n\n    @classmethod\n    def trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from arff trace.\n\n        Creates a trace file from arff object (for example, generated by a\n        local run).\n\n        Parameters\n        ----------\n        arff_obj : dict\n            LIAC arff obj, dict containing attributes, relation, data.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        attributes = arff_obj[\"attributes\"]\n        content = arff_obj[\"data\"]\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not supported for arff serialization\",\n        )\n\n    @classmethod\n    def _trace_from_arff_struct(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n        error_message: str,\n    ) -&gt; Self:\n        \"\"\"Generate a trace dictionary from ARFF structure.\n\n        Parameters\n        ----------\n        cls : type\n            The trace object to be created.\n        attributes : list[tuple[str, str]]\n            Attribute descriptions.\n        content : list[list[int | float | str]]]\n            List of instances.\n        error_message : str\n            Error message to raise if `setup_string` is in `attributes`.\n\n        Returns\n        -------\n        OrderedDict\n            A dictionary representing the trace.\n        \"\"\"\n        trace = OrderedDict()\n        attribute_idx = {att[0]: idx for idx, att in enumerate(attributes)}\n\n        for required_attribute in REQUIRED_ATTRIBUTES:\n            if required_attribute not in attribute_idx:\n                raise ValueError(\"arff misses required attribute: %s\" % required_attribute)\n        if \"setup_string\" in attribute_idx:\n            raise ValueError(error_message)\n\n        # note that the required attributes can not be duplicated because\n        # they are not parameters\n        parameter_attributes = []\n        for attribute in attribute_idx:\n            if attribute in REQUIRED_ATTRIBUTES or attribute == \"setup_string\":\n                continue\n\n            if not attribute.startswith(PREFIX):\n                raise ValueError(\n                    f\"Encountered unknown attribute {attribute} that does not start \"\n                    f\"with prefix {PREFIX}\",\n                )\n\n            parameter_attributes.append(attribute)\n\n        for itt in content:\n            repeat = int(itt[attribute_idx[\"repeat\"]])\n            fold = int(itt[attribute_idx[\"fold\"]])\n            iteration = int(itt[attribute_idx[\"iteration\"]])\n            evaluation = float(itt[attribute_idx[\"evaluation\"]])\n            selected_value = itt[attribute_idx[\"selected\"]]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for selected field, '\n                    \"received: %s\" % selected_value,\n                )\n\n            parameters = {\n                attribute: itt[attribute_idx[attribute]] for attribute in parameter_attributes\n            }\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=None,\n                evaluation=evaluation,\n                selected=selected,\n                parameters=parameters,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(None, trace)\n\n    @classmethod\n    def trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from xml.\n\n        Creates a trace file from the xml description.\n\n        Parameters\n        ----------\n        xml : string | file-like object\n            An xml description that can be either a `string` or a file-like\n            object.\n\n        Returns\n        -------\n        run : OpenMLRunTrace\n            Object containing the run id and a dict containing the trace\n            iterations.\n        \"\"\"\n        if isinstance(xml, Path):\n            xml = str(xml.absolute())\n\n        result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n        run_id = result_dict[\"oml:run_id\"]\n        trace = OrderedDict()\n\n        if \"oml:trace_iteration\" not in result_dict:\n            raise ValueError(\"Run does not contain valid trace. \")\n        if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n            raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n        for itt in result_dict[\"oml:trace_iteration\"]:\n            repeat = int(itt[\"oml:repeat\"])\n            fold = int(itt[\"oml:fold\"])\n            iteration = int(itt[\"oml:iteration\"])\n            setup_string = json.loads(itt[\"oml:setup_string\"])\n            evaluation = float(itt[\"oml:evaluation\"])\n            selected_value = itt[\"oml:selected\"]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for '\n                    \"selected field, received: %s\" % selected_value,\n                )\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=setup_string,\n                evaluation=evaluation,\n                selected=selected,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(run_id, trace)\n\n    @classmethod\n    def merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n        \"\"\"Merge multiple traces into a single trace.\n\n        Parameters\n        ----------\n        cls : type\n            Type of the trace object to be created.\n        traces : List[OpenMLRunTrace]\n            List of traces to merge.\n\n        Returns\n        -------\n        OpenMLRunTrace\n            A trace object representing the merged traces.\n\n        Raises\n        ------\n        ValueError\n            If the parameters in the iterations of the traces being merged are not equal.\n            If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n        \"\"\"\n        merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n        previous_iteration = None\n        for trace in traces:\n            for iteration in trace:\n                key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n                assert iteration.parameters is not None\n                param_keys = iteration.parameters.keys()\n\n                if previous_iteration is not None:\n                    trace_itr = merged_trace[previous_iteration]\n\n                    assert trace_itr.parameters is not None\n                    trace_itr_keys = trace_itr.parameters.keys()\n\n                    if list(param_keys) != list(trace_itr_keys):\n                        raise ValueError(\n                            \"Cannot merge traces because the parameters are not equal: \"\n                            \"{} vs {}\".format(\n                                list(trace_itr.parameters.keys()),\n                                list(iteration.parameters.keys()),\n                            ),\n                        )\n\n                if key in merged_trace:\n                    raise ValueError(\n                        f\"Cannot merge traces because key '{key}' was encountered twice\",\n                    )\n\n                merged_trace[key] = iteration\n                previous_iteration = key\n\n        return cls(None, merged_trace)\n\n    def __repr__(self) -&gt; str:\n        return \"[Run id: {}, {} trace iterations]\".format(\n            -1 if self.run_id is None else self.run_id,\n            len(self.trace_iterations),\n        )\n\n    def __iter__(self) -&gt; Iterator[OpenMLTraceIteration]:\n        yield from self.trace_iterations.values()\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.__init__","title":"<code>__init__(run_id, trace_iterations)</code>","text":"<p>Object to hold the trace content of a run.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Id for which the trace content is to be stored.</p> required <code>trace_iterations</code> <code>List[List]</code> <p>The trace content obtained by running a flow on a task.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>def __init__(\n    self,\n    run_id: int | None,\n    trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n):\n    \"\"\"Object to hold the trace content of a run.\n\n    Parameters\n    ----------\n    run_id : int\n        Id for which the trace content is to be stored.\n    trace_iterations : List[List]\n        The trace content obtained by running a flow on a task.\n    \"\"\"\n    self.run_id = run_id\n    self.trace_iterations = trace_iterations\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.generate","title":"<code>generate(attributes, content)</code>  <code>classmethod</code>","text":"<p>Generates an OpenMLRunTrace.</p> <p>Generates the trace object from the attributes and content extracted while running the underlying flow.</p> <p>Parameters:</p> Name Type Description Default <code>attributes</code> <code>list</code> <p>List of tuples describing the arff attributes.</p> required <code>content</code> <code>list</code> <p>List of lists containing information about the individual tuning runs.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef generate(\n    cls,\n    attributes: list[tuple[str, str]],\n    content: list[list[int | float | str]],\n) -&gt; OpenMLRunTrace:\n    \"\"\"Generates an OpenMLRunTrace.\n\n    Generates the trace object from the attributes and content extracted\n    while running the underlying flow.\n\n    Parameters\n    ----------\n    attributes : list\n        List of tuples describing the arff attributes.\n\n    content : list\n        List of lists containing information about the individual tuning\n        runs.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    if content is None:\n        raise ValueError(\"Trace content not available.\")\n    if attributes is None:\n        raise ValueError(\"Trace attributes not available.\")\n    if len(content) == 0:\n        raise ValueError(\"Trace content is empty.\")\n    if len(attributes) != len(content[0]):\n        raise ValueError(\n            \"Trace_attributes and trace_content not compatible:\"\n            f\" {attributes} vs {content[0]}\",\n        )\n\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not allowed when constructing a \"\n        \"trace object from run results.\",\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.get_selected_iteration","title":"<code>get_selected_iteration(fold, repeat)</code>","text":"<p>Returns the trace iteration that was marked as selected. In case multiple are marked as selected (should not happen) the first of these is returned</p> <p>Parameters:</p> Name Type Description Default <code>fold</code> <code>int</code> required <code>repeat</code> <code>int</code> required <p>Returns:</p> Type Description <code>int</code> <p>The trace iteration from the given fold and repeat that was selected as the best iteration by the search procedure</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n    \"\"\"\n    Returns the trace iteration that was marked as selected. In\n    case multiple are marked as selected (should not happen) the\n    first of these is returned\n\n    Parameters\n    ----------\n    fold: int\n\n    repeat: int\n\n    Returns\n    -------\n    int\n        The trace iteration from the given fold and repeat that was\n        selected as the best iteration by the search procedure\n    \"\"\"\n    for r, f, i in self.trace_iterations:\n        if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n            return i\n    raise ValueError(\n        \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.merge_traces","title":"<code>merge_traces(traces)</code>  <code>classmethod</code>","text":"<p>Merge multiple traces into a single trace.</p> <p>Parameters:</p> Name Type Description Default <code>cls</code> <code>type</code> <p>Type of the trace object to be created.</p> required <code>traces</code> <code>List[OpenMLRunTrace]</code> <p>List of traces to merge.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> <p>A trace object representing the merged traces.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the parameters in the iterations of the traces being merged are not equal. If a key (repeat, fold, iteration) is encountered twice while merging the traces.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n    \"\"\"Merge multiple traces into a single trace.\n\n    Parameters\n    ----------\n    cls : type\n        Type of the trace object to be created.\n    traces : List[OpenMLRunTrace]\n        List of traces to merge.\n\n    Returns\n    -------\n    OpenMLRunTrace\n        A trace object representing the merged traces.\n\n    Raises\n    ------\n    ValueError\n        If the parameters in the iterations of the traces being merged are not equal.\n        If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n    \"\"\"\n    merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n    previous_iteration = None\n    for trace in traces:\n        for iteration in trace:\n            key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n            assert iteration.parameters is not None\n            param_keys = iteration.parameters.keys()\n\n            if previous_iteration is not None:\n                trace_itr = merged_trace[previous_iteration]\n\n                assert trace_itr.parameters is not None\n                trace_itr_keys = trace_itr.parameters.keys()\n\n                if list(param_keys) != list(trace_itr_keys):\n                    raise ValueError(\n                        \"Cannot merge traces because the parameters are not equal: \"\n                        \"{} vs {}\".format(\n                            list(trace_itr.parameters.keys()),\n                            list(iteration.parameters.keys()),\n                        ),\n                    )\n\n            if key in merged_trace:\n                raise ValueError(\n                    f\"Cannot merge traces because key '{key}' was encountered twice\",\n                )\n\n            merged_trace[key] = iteration\n            previous_iteration = key\n\n    return cls(None, merged_trace)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.trace_from_arff","title":"<code>trace_from_arff(arff_obj)</code>  <code>classmethod</code>","text":"<p>Generate trace from arff trace.</p> <p>Creates a trace file from arff object (for example, generated by a local run).</p> <p>Parameters:</p> Name Type Description Default <code>arff_obj</code> <code>dict</code> <p>LIAC arff obj, dict containing attributes, relation, data.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from arff trace.\n\n    Creates a trace file from arff object (for example, generated by a\n    local run).\n\n    Parameters\n    ----------\n    arff_obj : dict\n        LIAC arff obj, dict containing attributes, relation, data.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    attributes = arff_obj[\"attributes\"]\n    content = arff_obj[\"data\"]\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not supported for arff serialization\",\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.trace_from_xml","title":"<code>trace_from_xml(xml)</code>  <code>classmethod</code>","text":"<p>Generate trace from xml.</p> <p>Creates a trace file from the xml description.</p> <p>Parameters:</p> Name Type Description Default <code>xml</code> <code>string | file-like object</code> <p>An xml description that can be either a <code>string</code> or a file-like object.</p> required <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRunTrace</code> <p>Object containing the run id and a dict containing the trace iterations.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from xml.\n\n    Creates a trace file from the xml description.\n\n    Parameters\n    ----------\n    xml : string | file-like object\n        An xml description that can be either a `string` or a file-like\n        object.\n\n    Returns\n    -------\n    run : OpenMLRunTrace\n        Object containing the run id and a dict containing the trace\n        iterations.\n    \"\"\"\n    if isinstance(xml, Path):\n        xml = str(xml.absolute())\n\n    result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n    run_id = result_dict[\"oml:run_id\"]\n    trace = OrderedDict()\n\n    if \"oml:trace_iteration\" not in result_dict:\n        raise ValueError(\"Run does not contain valid trace. \")\n    if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n        raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n    for itt in result_dict[\"oml:trace_iteration\"]:\n        repeat = int(itt[\"oml:repeat\"])\n        fold = int(itt[\"oml:fold\"])\n        iteration = int(itt[\"oml:iteration\"])\n        setup_string = json.loads(itt[\"oml:setup_string\"])\n        evaluation = float(itt[\"oml:evaluation\"])\n        selected_value = itt[\"oml:selected\"]\n        if selected_value == \"true\":\n            selected = True\n        elif selected_value == \"false\":\n            selected = False\n        else:\n            raise ValueError(\n                'expected {\"true\", \"false\"} value for '\n                \"selected field, received: %s\" % selected_value,\n            )\n\n        current = OpenMLTraceIteration(\n            repeat=repeat,\n            fold=fold,\n            iteration=iteration,\n            setup_string=setup_string,\n            evaluation=evaluation,\n            selected=selected,\n        )\n        trace[(repeat, fold, iteration)] = current\n\n    return cls(run_id, trace)\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLRunTrace.trace_to_arff","title":"<code>trace_to_arff()</code>","text":"<p>Generate the arff dictionary for uploading predictions to the server.</p> <p>Uses the trace object to generate an arff dictionary representation.</p> <p>Returns:</p> Name Type Description <code>arff_dict</code> <code>dict</code> <p>Dictionary representation of the ARFF file that will be uploaded. Contains information about the optimization trace.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def trace_to_arff(self) -&gt; dict[str, Any]:\n    \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n    Uses the trace object to generate an arff dictionary representation.\n\n    Returns\n    -------\n    arff_dict : dict\n        Dictionary representation of the ARFF file that will be uploaded.\n        Contains information about the optimization trace.\n    \"\"\"\n    if self.trace_iterations is None:\n        raise ValueError(\"trace_iterations missing from the trace object\")\n\n    # attributes that will be in trace arff\n    trace_attributes = [\n        (\"repeat\", \"NUMERIC\"),\n        (\"fold\", \"NUMERIC\"),\n        (\"iteration\", \"NUMERIC\"),\n        (\"evaluation\", \"NUMERIC\"),\n        (\"selected\", [\"true\", \"false\"]),\n    ]\n    trace_attributes.extend(\n        [\n            (PREFIX + parameter, \"STRING\")\n            for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n        ],\n    )\n\n    arff_dict: dict[str, Any] = {}\n    data = []\n    for trace_iteration in self.trace_iterations.values():\n        tmp_list = []\n        for _attr, _ in trace_attributes:\n            if _attr.startswith(PREFIX):\n                attr = _attr[len(PREFIX) :]\n                value = trace_iteration.get_parameters()[attr]\n            else:\n                attr = _attr\n                value = getattr(trace_iteration, attr)\n\n            if attr == \"selected\":\n                tmp_list.append(\"true\" if value else \"false\")\n            else:\n                tmp_list.append(value)\n        data.append(tmp_list)\n\n    arff_dict[\"attributes\"] = trace_attributes\n    arff_dict[\"data\"] = data\n    # TODO allow to pass a trace description when running a flow\n    arff_dict[\"relation\"] = \"Trace\"\n    return arff_dict\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLTraceIteration","title":"<code>OpenMLTraceIteration</code>  <code>dataclass</code>","text":"<p>OpenML Trace Iteration: parsed output from Run Trace call Exactly one of <code>setup_string</code> or <code>parameters</code> must be provided.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>repeat number (in case of no repeats: 0)</p> required <code>fold</code> <code>int</code> <p>fold number (in case of no folds: 0)</p> required <code>iteration</code> <code>int</code> <p>iteration number of optimization procedure</p> required <code>setup_string</code> <code>str</code> <p>json string representing the parameters If not provided, <code>parameters</code> should be set.</p> <code>None</code> <code>evaluation</code> <code>double</code> <p>The evaluation that was awarded to this trace iteration. Measure is defined by the task</p> required <code>selected</code> <code>bool</code> <p>Whether this was the best of all iterations, and hence selected for making predictions. Per fold/repeat there should be only one iteration selected</p> required <code>parameters</code> <code>OrderedDict</code> <p>Dictionary specifying parameter names and their values. If not provided, <code>setup_string</code> should be set.</p> <code>None</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@dataclass\nclass OpenMLTraceIteration:\n    \"\"\"\n    OpenML Trace Iteration: parsed output from Run Trace call\n    Exactly one of `setup_string` or `parameters` must be provided.\n\n    Parameters\n    ----------\n    repeat : int\n        repeat number (in case of no repeats: 0)\n\n    fold : int\n        fold number (in case of no folds: 0)\n\n    iteration : int\n        iteration number of optimization procedure\n\n    setup_string : str, optional\n        json string representing the parameters\n        If not provided, ``parameters`` should be set.\n\n    evaluation : double\n        The evaluation that was awarded to this trace iteration.\n        Measure is defined by the task\n\n    selected : bool\n        Whether this was the best of all iterations, and hence\n        selected for making predictions. Per fold/repeat there\n        should be only one iteration selected\n\n    parameters : OrderedDict, optional\n        Dictionary specifying parameter names and their values.\n        If not provided, ``setup_string`` should be set.\n    \"\"\"\n\n    repeat: int\n    fold: int\n    iteration: int\n\n    evaluation: float\n    selected: bool\n\n    setup_string: dict[str, str] | None = None\n    parameters: dict[str, str | int | float] | None = None\n\n    def __post_init__(self) -&gt; None:\n        # TODO: refactor into one argument of type &lt;str | OrderedDict&gt;\n        if self.setup_string and self.parameters:\n            raise ValueError(\n                \"Can only be instantiated with either `setup_string` or `parameters` argument.\",\n            )\n\n        if not (self.setup_string or self.parameters):\n            raise ValueError(\n                \"Either `setup_string` or `parameters` needs to be passed as argument.\",\n            )\n\n        if self.parameters is not None and not isinstance(self.parameters, dict):\n            raise TypeError(\n                \"argument parameters is not an instance of OrderedDict, but %s\"\n                % str(type(self.parameters)),\n            )\n\n    def get_parameters(self) -&gt; dict[str, Any]:\n        \"\"\"Get the parameters of this trace iteration.\"\"\"\n        # parameters have prefix 'parameter_'\n        if self.setup_string:\n            return {\n                param[len(PREFIX) :]: json.loads(value)\n                for param, value in self.setup_string.items()\n            }\n\n        assert self.parameters is not None\n        return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/runs/#openml.runs.OpenMLTraceIteration.get_parameters","title":"<code>get_parameters()</code>","text":"<p>Get the parameters of this trace iteration.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_parameters(self) -&gt; dict[str, Any]:\n    \"\"\"Get the parameters of this trace iteration.\"\"\"\n    # parameters have prefix 'parameter_'\n    if self.setup_string:\n        return {\n            param[len(PREFIX) :]: json.loads(value)\n            for param, value in self.setup_string.items()\n        }\n\n    assert self.parameters is not None\n    return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/runs/#openml.runs.delete_run","title":"<code>delete_run(run_id)</code>","text":"<p>Delete run with id <code>run_id</code> from the OpenML server.</p> <p>You can only delete runs which you uploaded.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML id of the run</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def delete_run(run_id: int) -&gt; bool:\n    \"\"\"Delete run with id `run_id` from the OpenML server.\n\n    You can only delete runs which you uploaded.\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML id of the run\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"run\", run_id)\n</code></pre>"},{"location":"reference/runs/#openml.runs.get_run","title":"<code>get_run(run_id, ignore_cache=False)</code>","text":"<p>Gets run corresponding to run_id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <code>ignore_cache</code> <code>bool</code> <p>Whether to ignore the cache. If <code>true</code> this will download and overwrite the run xml even if the requested run is already cached.</p> <code>False</code> <code>ignore_cache</code> <code>bool</code> <code>False</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Run corresponding to ID, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_run(run_id: int, ignore_cache: bool = False) -&gt; OpenMLRun:  # noqa: FBT002, FBT001\n    \"\"\"Gets run corresponding to run_id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    ignore_cache : bool\n        Whether to ignore the cache. If ``true`` this will download and overwrite the run xml\n        even if the requested run is already cached.\n\n    ignore_cache\n\n    Returns\n    -------\n    run : OpenMLRun\n        Run corresponding to ID, fetched from the server.\n    \"\"\"\n    run_dir = Path(openml.utils._create_cache_directory_for_id(RUNS_CACHE_DIR_NAME, run_id))\n    run_file = run_dir / \"description.xml\"\n\n    run_dir.mkdir(parents=True, exist_ok=True)\n\n    try:\n        if not ignore_cache:\n            return _get_cached_run(run_id)\n\n        raise OpenMLCacheException(message=\"dummy\")\n\n    except OpenMLCacheException:\n        run_xml = openml._api_calls._perform_api_call(\"run/%d\" % run_id, \"get\")\n        with run_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(run_xml)\n\n    return _create_run_from_xml(run_xml)\n</code></pre>"},{"location":"reference/runs/#openml.runs.get_run_trace","title":"<code>get_run_trace(run_id)</code>","text":"<p>Get the optimization trace object for a given run id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>OpenMLTrace</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_run_trace(run_id: int) -&gt; OpenMLRunTrace:\n    \"\"\"\n    Get the optimization trace object for a given run id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    Returns\n    -------\n    openml.runs.OpenMLTrace\n    \"\"\"\n    trace_xml = openml._api_calls._perform_api_call(\"run/trace/%d\" % run_id, \"get\")\n    return OpenMLRunTrace.trace_from_xml(trace_xml)\n</code></pre>"},{"location":"reference/runs/#openml.runs.get_runs","title":"<code>get_runs(run_ids)</code>","text":"<p>Gets all runs in run_ids list.</p> <p>Parameters:</p> Name Type Description Default <code>run_ids</code> <code>list of ints</code> required <p>Returns:</p> Name Type Description <code>runs</code> <code>list of OpenMLRun</code> <p>List of runs corresponding to IDs, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_runs(run_ids: list[int]) -&gt; list[OpenMLRun]:\n    \"\"\"Gets all runs in run_ids list.\n\n    Parameters\n    ----------\n    run_ids : list of ints\n\n    Returns\n    -------\n    runs : list of OpenMLRun\n        List of runs corresponding to IDs, fetched from the server.\n    \"\"\"\n    runs = []\n    for run_id in run_ids:\n        runs.append(get_run(run_id))\n    return runs\n</code></pre>"},{"location":"reference/runs/#openml.runs.initialize_model_from_run","title":"<code>initialize_model_from_run(run_id)</code>","text":"<p>Initialized a model based on a run_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_run(run_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a run_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.setup_id is not None\n    return initialize_model(run.setup_id)\n</code></pre>"},{"location":"reference/runs/#openml.runs.initialize_model_from_trace","title":"<code>initialize_model_from_trace(run_id, repeat, fold, iteration=None)</code>","text":"<p>Initialize a model based on the parameters that were set by an optimization procedure (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id. Should contain a trace file, otherwise a OpenMLServerException is raised</p> required <code>repeat</code> <code>int</code> <p>The repeat nr (column in trace file)</p> required <code>fold</code> <code>int</code> <p>The fold nr (column in trace file)</p> required <code>iteration</code> <code>int</code> <p>The iteration nr (column in trace file). If None, the best (selected) iteration will be searched (slow), according to the selection criteria implemented in OpenMLRunTrace.get_selected_iteration</p> <code>None</code> <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_trace(\n    run_id: int,\n    repeat: int,\n    fold: int,\n    iteration: int | None = None,\n) -&gt; Any:\n    \"\"\"\n    Initialize a model based on the parameters that were set\n    by an optimization procedure (i.e., using the exact same\n    parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id. Should contain a trace file,\n        otherwise a OpenMLServerException is raised\n\n    repeat : int\n        The repeat nr (column in trace file)\n\n    fold : int\n        The fold nr (column in trace file)\n\n    iteration : int\n        The iteration nr (column in trace file). If None, the\n        best (selected) iteration will be searched (slow),\n        according to the selection criteria implemented in\n        OpenMLRunTrace.get_selected_iteration\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.flow_id is not None\n\n    flow = get_flow(run.flow_id)\n    run_trace = get_run_trace(run_id)\n\n    if iteration is None:\n        iteration = run_trace.get_selected_iteration(repeat, fold)\n\n    request = (repeat, fold, iteration)\n    if request not in run_trace.trace_iterations:\n        raise ValueError(\"Combination repeat, fold, iteration not available\")\n    current = run_trace.trace_iterations[(repeat, fold, iteration)]\n\n    search_model = initialize_model_from_run(run_id)\n    return flow.extension.instantiate_model_from_hpo_class(search_model, current)\n</code></pre>"},{"location":"reference/runs/#openml.runs.list_runs","title":"<code>list_runs(offset=None, size=None, id=None, task=None, setup=None, flow=None, uploader=None, tag=None, study=None, display_errors=False, output_format='dict', **kwargs)</code>","text":"<p>List all runs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>id</code> <code>list</code> <code>None</code> <code>task</code> <code>list</code> <code>None</code> <code>setup</code> <code>list | None</code> <code>None</code> <code>flow</code> <code>list</code> <code>None</code> <code>uploader</code> <code>list</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>display_errors</code> <code>(bool, optional(default=None))</code> <p>Whether to list runs which have an error (for example a missing prediction file).</p> <code>False</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators: task_type.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict of dicts, or dataframe</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def list_runs(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    id: list | None = None,  # noqa: A002\n    task: list[int] | None = None,\n    setup: list | None = None,\n    flow: list | None = None,\n    uploader: list | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    display_errors: bool = False,  # noqa: FBT001, FBT002\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all runs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n\n    id : list, optional\n\n    task : list, optional\n\n    setup: list, optional\n\n    flow : list, optional\n\n    uploader : list, optional\n\n    tag : str, optional\n\n    study : int, optional\n\n    display_errors : bool, optional (default=None)\n        Whether to list runs which have an error (for example a missing\n        prediction file).\n\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    kwargs : dict, optional\n        Legal filter operators: task_type.\n\n    Returns\n    -------\n    dict of dicts, or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\"Invalid output format selected. Only 'dict' or 'dataframe' applicable.\")\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    # TODO(eddiebergman): Do we really need this runtime type validation?\n    if id is not None and (not isinstance(id, list)):\n        raise TypeError(\"id must be of type list.\")\n    if task is not None and (not isinstance(task, list)):\n        raise TypeError(\"task must be of type list.\")\n    if setup is not None and (not isinstance(setup, list)):\n        raise TypeError(\"setup must be of type list.\")\n    if flow is not None and (not isinstance(flow, list)):\n        raise TypeError(\"flow must be of type list.\")\n    if uploader is not None and (not isinstance(uploader, list)):\n        raise TypeError(\"uploader must be of type list.\")\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_runs,\n        offset=offset,\n        size=size,\n        id=id,\n        task=task,\n        setup=setup,\n        flow=flow,\n        uploader=uploader,\n        tag=tag,\n        study=study,\n        display_errors=display_errors,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/runs/#openml.runs.run_exists","title":"<code>run_exists(task_id, setup_id)</code>","text":"<p>Checks whether a task/setup combination is already present on the server.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> required <code>setup_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>    Set run ids for runs where flow setup_id was run on task_id. Empty</code> <p>set if it wasn't run yet.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_exists(task_id: int, setup_id: int) -&gt; set[int]:\n    \"\"\"Checks whether a task/setup combination is already present on the\n    server.\n\n    Parameters\n    ----------\n    task_id : int\n\n    setup_id : int\n\n    Returns\n    -------\n        Set run ids for runs where flow setup_id was run on task_id. Empty\n        set if it wasn't run yet.\n    \"\"\"\n    if setup_id &lt;= 0:\n        # openml setups are in range 1-inf\n        return set()\n\n    try:\n        result = list_runs(task=[task_id], setup=[setup_id], output_format=\"dataframe\")\n        assert isinstance(result, pd.DataFrame)  # TODO(eddiebergman): Remove once #1299\n        return set() if result.empty else set(result[\"run_id\"])\n    except OpenMLServerException as exception:\n        # error code implies no results. The run does not exist yet\n        if exception.code != ERROR_CODE:\n            raise exception\n        return set()\n</code></pre>"},{"location":"reference/runs/#openml.runs.run_flow_on_task","title":"<code>run_flow_on_task(flow, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model provided by the flow on the dataset defined by task.</p> <p>Takes the flow and repeat information into account. The Flow may optionally be published.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>A flow wraps a machine learning model together with relevant information. The model has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask</code> <p>Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_flow_on_task(  # noqa: C901, PLR0912, PLR0915, PLR0913\n    flow: OpenMLFlow,\n    task: OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT002, FBT001\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun:\n    \"\"\"Run the model provided by the flow on the dataset defined by task.\n\n    Takes the flow and repeat information into account.\n    The Flow may optionally be published.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        A flow wraps a machine learning model together with relevant information.\n        The model has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask\n        Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    \"\"\"\n    if flow_tags is not None and not isinstance(flow_tags, list):\n        raise ValueError(\"flow_tags should be a list\")\n\n    # TODO: At some point in the future do not allow for arguments in old order (changed 6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    if isinstance(flow, OpenMLTask) and isinstance(task, OpenMLFlow):\n        # We want to allow either order of argument (to avoid confusion).\n        warnings.warn(\n            \"The old argument order (Flow, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, Flow).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, flow = flow, task\n\n    if task.task_id is None:\n        raise ValueError(\"The task should be published at OpenML\")\n\n    if flow.model is None:\n        flow.model = flow.extension.flow_to_model(flow)\n\n    flow.model = flow.extension.seed_model(flow.model, seed=seed)\n\n    # We only need to sync with the server right now if we want to upload the flow,\n    # or ensure no duplicate runs exist. Otherwise it can be synced at upload time.\n    flow_id = None\n    if upload_flow or avoid_duplicate_runs:\n        flow_id = flow_exists(flow.name, flow.external_version)\n        if isinstance(flow.flow_id, int) and flow_id != flow.flow_id:\n            if flow_id is not False:\n                raise PyOpenMLError(\n                    \"Local flow_id does not match server flow_id: \"\n                    f\"'{flow.flow_id}' vs '{flow_id}'\",\n                )\n            raise PyOpenMLError(\n                \"Flow does not exist on the server, but 'flow.flow_id' is not None.\"\n            )\n        if upload_flow and flow_id is False:\n            flow.publish()\n            flow_id = flow.flow_id\n        elif flow_id:\n            flow_from_server = get_flow(flow_id)\n            _copy_server_fields(flow_from_server, flow)\n            if avoid_duplicate_runs:\n                flow_from_server.model = flow.model\n                setup_id = setup_exists(flow_from_server)\n                ids = run_exists(task.task_id, setup_id)\n                if ids:\n                    error_message = (\n                        \"One or more runs of this setup were already performed on the task.\"\n                    )\n                    raise OpenMLRunsExistError(ids, error_message)\n        else:\n            # Flow does not exist on server and we do not want to upload it.\n            # No sync with the server happens.\n            flow_id = None\n\n    dataset = task.get_dataset()\n\n    run_environment = flow.extension.get_version_information()\n    tags = [\"openml-python\", run_environment[1]]\n\n    if flow.extension.check_if_model_fitted(flow.model):\n        warnings.warn(\n            \"The model is already fitted!\"\n            \" This might cause inconsistency in comparison of results.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # execute the run\n    res = _run_task_get_arffcontent(\n        model=flow.model,\n        task=task,\n        extension=flow.extension,\n        add_local_measures=add_local_measures,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n\n    data_content, trace, fold_evaluations, sample_evaluations = res\n    fields = [*run_environment, time.strftime(\"%c\"), \"Created by run_flow_on_task\"]\n    generated_description = \"\\n\".join(fields)\n    run = OpenMLRun(\n        task_id=task.task_id,\n        flow_id=flow_id,\n        dataset_id=dataset.dataset_id,\n        model=flow.model,\n        flow_name=flow.name,\n        tags=tags,\n        trace=trace,\n        data_content=data_content,\n        flow=flow,\n        setup_string=flow.extension.create_setup_string(flow.model),\n        description_text=generated_description,\n    )\n\n    if (upload_flow or avoid_duplicate_runs) and flow.flow_id is not None:\n        # We only extract the parameter settings if a sync happened with the server.\n        # I.e. when the flow was uploaded or we found it in the avoid_duplicate check.\n        # Otherwise, we will do this at upload time.\n        run.parameter_settings = flow.extension.obtain_parameter_values(flow)\n\n    # now we need to attach the detailed evaluations\n    if task.task_type_id == TaskType.LEARNING_CURVE:\n        run.sample_evaluations = sample_evaluations\n    else:\n        run.fold_evaluations = fold_evaluations\n\n    if flow_id:\n        message = f\"Executed Task {task.task_id} with Flow id:{run.flow_id}\"\n    else:\n        message = f\"Executed Task {task.task_id} on local Flow with name {flow.name}.\"\n    config.logger.info(message)\n\n    return run\n</code></pre>"},{"location":"reference/runs/#openml.runs.run_model_on_task","title":"<code>run_model_on_task(model, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, return_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model on the dataset defined by the task.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>A model which has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask or int or str</code> <p>Task to perform or Task id. This may be a model instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>return_flow</code> <code>bool(default=False)</code> <p>If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> <code>flow</code> <code>OpenMLFlow (optional, only if `return_flow` is True).</code> <p>Flow generated from the model.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_model_on_task(  # noqa: PLR0913\n    model: Any,\n    task: int | str | OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT001, FBT002\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    return_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun | tuple[OpenMLRun, OpenMLFlow]:\n    \"\"\"Run the model on the dataset defined by the task.\n\n    Parameters\n    ----------\n    model : sklearn model\n        A model which has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask or int or str\n        Task to perform or Task id.\n        This may be a model instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    return_flow : bool (default=False)\n        If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    flow : OpenMLFlow (optional, only if `return_flow` is True).\n        Flow generated from the model.\n    \"\"\"\n    if avoid_duplicate_runs and not config.apikey:\n        warnings.warn(\n            \"avoid_duplicate_runs is set to True, but no API key is set. \"\n            \"Please set your API key in the OpenML configuration file, see\"\n            \"https://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial\"\n            \".html#authentication for more information on authentication.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # TODO: At some point in the future do not allow for arguments in old order (6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    # When removing this please also remove the method `is_estimator` from the extension\n    # interface as it is only used here (MF, 3-2019)\n    if isinstance(model, (int, str, OpenMLTask)):\n        warnings.warn(\n            \"The old argument order (task, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, task).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, model = model, task\n\n    extension = get_extension_by_model(model, raise_if_no_extension=True)\n    if extension is None:\n        # This should never happen and is only here to please mypy will be gone soon once the\n        # whole function is removed\n        raise TypeError(extension)\n\n    flow = extension.model_to_flow(model)\n\n    def get_task_and_type_conversion(_task: int | str | OpenMLTask) -&gt; OpenMLTask:\n        \"\"\"Retrieve an OpenMLTask object from either an integer or string ID,\n        or directly from an OpenMLTask object.\n\n        Parameters\n        ----------\n        _task : Union[int, str, OpenMLTask]\n            The task ID or the OpenMLTask object.\n\n        Returns\n        -------\n        OpenMLTask\n            The OpenMLTask object.\n        \"\"\"\n        if isinstance(_task, (int, str)):\n            return get_task(int(_task))  # type: ignore\n\n        return _task\n\n    task = get_task_and_type_conversion(task)\n\n    run = run_flow_on_task(\n        task=task,\n        flow=flow,\n        avoid_duplicate_runs=avoid_duplicate_runs,\n        flow_tags=flow_tags,\n        seed=seed,\n        add_local_measures=add_local_measures,\n        upload_flow=upload_flow,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n    if return_flow:\n        return run, flow\n    return run\n</code></pre>"},{"location":"reference/runs/functions/","title":"functions","text":""},{"location":"reference/runs/functions/#openml.runs.functions.__list_runs","title":"<code>__list_runs(api_call, output_format='dict')</code>","text":"<p>Helper function to parse API calls which are lists of runs</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def __list_runs(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\"] = \"dict\"\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Helper function to parse API calls which are lists of runs\"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    runs_dict = xmltodict.parse(xml_string, force_list=(\"oml:run\",))\n    # Minimalistic check if the XML is useful\n    if \"oml:runs\" not in runs_dict:\n        raise ValueError(f'Error in return XML, does not contain \"oml:runs\": {runs_dict}')\n\n    if \"@xmlns:oml\" not in runs_dict[\"oml:runs\"]:\n        raise ValueError(\n            f'Error in return XML, does not contain \"oml:runs\"/@xmlns:oml: {runs_dict}'\n        )\n\n    if runs_dict[\"oml:runs\"][\"@xmlns:oml\"] != \"http://openml.org/openml\":\n        raise ValueError(\n            \"Error in return XML, value of  \"\n            '\"oml:runs\"/@xmlns:oml is not '\n            f'\"http://openml.org/openml\": {runs_dict}',\n        )\n\n    assert isinstance(runs_dict[\"oml:runs\"][\"oml:run\"], list), type(runs_dict[\"oml:runs\"])\n\n    runs = {\n        int(r[\"oml:run_id\"]): {\n            \"run_id\": int(r[\"oml:run_id\"]),\n            \"task_id\": int(r[\"oml:task_id\"]),\n            \"setup_id\": int(r[\"oml:setup_id\"]),\n            \"flow_id\": int(r[\"oml:flow_id\"]),\n            \"uploader\": int(r[\"oml:uploader\"]),\n            \"task_type\": TaskType(int(r[\"oml:task_type_id\"])),\n            \"upload_time\": str(r[\"oml:upload_time\"]),\n            \"error_message\": str((r[\"oml:error_message\"]) or \"\"),\n        }\n        for r in runs_dict[\"oml:runs\"][\"oml:run\"]\n    }\n\n    if output_format == \"dataframe\":\n        runs = pd.DataFrame.from_dict(runs, orient=\"index\")\n\n    return runs\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.delete_run","title":"<code>delete_run(run_id)</code>","text":"<p>Delete run with id <code>run_id</code> from the OpenML server.</p> <p>You can only delete runs which you uploaded.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML id of the run</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def delete_run(run_id: int) -&gt; bool:\n    \"\"\"Delete run with id `run_id` from the OpenML server.\n\n    You can only delete runs which you uploaded.\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML id of the run\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"run\", run_id)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.format_prediction","title":"<code>format_prediction(task, repeat, fold, index, prediction, truth, sample=None, proba=None)</code>","text":"<p>Format the predictions in the specific order as required for the run results.</p> <p>Parameters:</p> Name Type Description Default <code>task</code> <code>OpenMLSupervisedTask</code> <p>Task for which to format the predictions.</p> required <code>repeat</code> <code>int</code> <p>From which repeat this predictions is made.</p> required <code>fold</code> <code>int</code> <p>From which fold this prediction is made.</p> required <code>index</code> <code>int</code> <p>For which index this prediction is made.</p> required <code>prediction</code> <code>str | int | float</code> <p>The predicted class label or value.</p> required <code>truth</code> <code>str | int | float</code> <p>The true class label or value.</p> required <code>sample</code> <code>int | None</code> <p>From which sample set this prediction is made. Required only for LearningCurve tasks.</p> <code>None</code> <code>proba</code> <code>dict[str, float] | None</code> <p>For classification tasks only. A mapping from each class label to their predicted probability. The dictionary should contain an entry for each of the <code>task.class_labels</code>. E.g.: {\"Iris-Setosa\": 0.2, \"Iris-Versicolor\": 0.7, \"Iris-Virginica\": 0.1}</p> <code>None</code> <p>Returns:</p> Type Description <code>A list with elements for the prediction results of a run.</code> <code>The returned order of the elements is (if available):</code> <p>[repeat, fold, sample, index, prediction, truth, *probabilities]</p> <code>This order follows the R Client API.</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def format_prediction(  # noqa: PLR0913\n    task: OpenMLSupervisedTask,\n    repeat: int,\n    fold: int,\n    index: int,\n    prediction: str | int | float,\n    truth: str | int | float,\n    sample: int | None = None,\n    proba: dict[str, float] | None = None,\n) -&gt; list[str | int | float]:\n    \"\"\"Format the predictions in the specific order as required for the run results.\n\n    Parameters\n    ----------\n    task: OpenMLSupervisedTask\n        Task for which to format the predictions.\n    repeat: int\n        From which repeat this predictions is made.\n    fold: int\n        From which fold this prediction is made.\n    index: int\n        For which index this prediction is made.\n    prediction: str, int or float\n        The predicted class label or value.\n    truth: str, int or float\n        The true class label or value.\n    sample: int, optional (default=None)\n        From which sample set this prediction is made.\n        Required only for LearningCurve tasks.\n    proba: Dict[str, float], optional (default=None)\n        For classification tasks only.\n        A mapping from each class label to their predicted probability.\n        The dictionary should contain an entry for each of the `task.class_labels`.\n        E.g.: {\"Iris-Setosa\": 0.2, \"Iris-Versicolor\": 0.7, \"Iris-Virginica\": 0.1}\n\n    Returns\n    -------\n    A list with elements for the prediction results of a run.\n\n    The returned order of the elements is (if available):\n        [repeat, fold, sample, index, prediction, truth, *probabilities]\n\n    This order follows the R Client API.\n    \"\"\"\n    if isinstance(task, OpenMLClassificationTask):\n        if proba is None:\n            raise ValueError(\"`proba` is required for classification task\")\n        if task.class_labels is None:\n            raise ValueError(\"The classification task must have class labels set\")\n        if not set(task.class_labels) == set(proba):\n            raise ValueError(\"Each class should have a predicted probability\")\n        if sample is None:\n            if isinstance(task, OpenMLLearningCurveTask):\n                raise ValueError(\"`sample` can not be none for LearningCurveTask\")\n\n            sample = 0\n        probabilities = [proba[c] for c in task.class_labels]\n        return [repeat, fold, sample, index, prediction, truth, *probabilities]\n\n    if isinstance(task, OpenMLRegressionTask):\n        return [repeat, fold, index, prediction, truth]\n\n    raise NotImplementedError(f\"Formatting for {type(task)} is not supported.\")\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.get_run","title":"<code>get_run(run_id, ignore_cache=False)</code>","text":"<p>Gets run corresponding to run_id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <code>ignore_cache</code> <code>bool</code> <p>Whether to ignore the cache. If <code>true</code> this will download and overwrite the run xml even if the requested run is already cached.</p> <code>False</code> <code>ignore_cache</code> <code>bool</code> <code>False</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Run corresponding to ID, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_run(run_id: int, ignore_cache: bool = False) -&gt; OpenMLRun:  # noqa: FBT002, FBT001\n    \"\"\"Gets run corresponding to run_id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    ignore_cache : bool\n        Whether to ignore the cache. If ``true`` this will download and overwrite the run xml\n        even if the requested run is already cached.\n\n    ignore_cache\n\n    Returns\n    -------\n    run : OpenMLRun\n        Run corresponding to ID, fetched from the server.\n    \"\"\"\n    run_dir = Path(openml.utils._create_cache_directory_for_id(RUNS_CACHE_DIR_NAME, run_id))\n    run_file = run_dir / \"description.xml\"\n\n    run_dir.mkdir(parents=True, exist_ok=True)\n\n    try:\n        if not ignore_cache:\n            return _get_cached_run(run_id)\n\n        raise OpenMLCacheException(message=\"dummy\")\n\n    except OpenMLCacheException:\n        run_xml = openml._api_calls._perform_api_call(\"run/%d\" % run_id, \"get\")\n        with run_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(run_xml)\n\n    return _create_run_from_xml(run_xml)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.get_run_trace","title":"<code>get_run_trace(run_id)</code>","text":"<p>Get the optimization trace object for a given run id.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>OpenMLTrace</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_run_trace(run_id: int) -&gt; OpenMLRunTrace:\n    \"\"\"\n    Get the optimization trace object for a given run id.\n\n    Parameters\n    ----------\n    run_id : int\n\n    Returns\n    -------\n    openml.runs.OpenMLTrace\n    \"\"\"\n    trace_xml = openml._api_calls._perform_api_call(\"run/trace/%d\" % run_id, \"get\")\n    return OpenMLRunTrace.trace_from_xml(trace_xml)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.get_runs","title":"<code>get_runs(run_ids)</code>","text":"<p>Gets all runs in run_ids list.</p> <p>Parameters:</p> Name Type Description Default <code>run_ids</code> <code>list of ints</code> required <p>Returns:</p> Name Type Description <code>runs</code> <code>list of OpenMLRun</code> <p>List of runs corresponding to IDs, fetched from the server.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def get_runs(run_ids: list[int]) -&gt; list[OpenMLRun]:\n    \"\"\"Gets all runs in run_ids list.\n\n    Parameters\n    ----------\n    run_ids : list of ints\n\n    Returns\n    -------\n    runs : list of OpenMLRun\n        List of runs corresponding to IDs, fetched from the server.\n    \"\"\"\n    runs = []\n    for run_id in run_ids:\n        runs.append(get_run(run_id))\n    return runs\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.initialize_model_from_run","title":"<code>initialize_model_from_run(run_id)</code>","text":"<p>Initialized a model based on a run_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_run(run_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a run_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.setup_id is not None\n    return initialize_model(run.setup_id)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.initialize_model_from_trace","title":"<code>initialize_model_from_trace(run_id, repeat, fold, iteration=None)</code>","text":"<p>Initialize a model based on the parameters that were set by an optimization procedure (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>The Openml run_id. Should contain a trace file, otherwise a OpenMLServerException is raised</p> required <code>repeat</code> <code>int</code> <p>The repeat nr (column in trace file)</p> required <code>fold</code> <code>int</code> <p>The fold nr (column in trace file)</p> required <code>iteration</code> <code>int</code> <p>The iteration nr (column in trace file). If None, the best (selected) iteration will be searched (slow), according to the selection criteria implemented in OpenMLRunTrace.get_selected_iteration</p> <code>None</code> <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def initialize_model_from_trace(\n    run_id: int,\n    repeat: int,\n    fold: int,\n    iteration: int | None = None,\n) -&gt; Any:\n    \"\"\"\n    Initialize a model based on the parameters that were set\n    by an optimization procedure (i.e., using the exact same\n    parameter settings)\n\n    Parameters\n    ----------\n    run_id : int\n        The Openml run_id. Should contain a trace file,\n        otherwise a OpenMLServerException is raised\n\n    repeat : int\n        The repeat nr (column in trace file)\n\n    fold : int\n        The fold nr (column in trace file)\n\n    iteration : int\n        The iteration nr (column in trace file). If None, the\n        best (selected) iteration will be searched (slow),\n        according to the selection criteria implemented in\n        OpenMLRunTrace.get_selected_iteration\n\n    Returns\n    -------\n    model\n    \"\"\"\n    run = get_run(run_id)\n    # TODO(eddiebergman): I imagine this is None if it's not published,\n    # might need to raise an explicit error for that\n    assert run.flow_id is not None\n\n    flow = get_flow(run.flow_id)\n    run_trace = get_run_trace(run_id)\n\n    if iteration is None:\n        iteration = run_trace.get_selected_iteration(repeat, fold)\n\n    request = (repeat, fold, iteration)\n    if request not in run_trace.trace_iterations:\n        raise ValueError(\"Combination repeat, fold, iteration not available\")\n    current = run_trace.trace_iterations[(repeat, fold, iteration)]\n\n    search_model = initialize_model_from_run(run_id)\n    return flow.extension.instantiate_model_from_hpo_class(search_model, current)\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.list_runs","title":"<code>list_runs(offset=None, size=None, id=None, task=None, setup=None, flow=None, uploader=None, tag=None, study=None, display_errors=False, output_format='dict', **kwargs)</code>","text":"<p>List all runs matching all of the given filters. (Supports large amount of results)</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>the number of runs to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of runs to show</p> <code>None</code> <code>id</code> <code>list</code> <code>None</code> <code>task</code> <code>list</code> <code>None</code> <code>setup</code> <code>list | None</code> <code>None</code> <code>flow</code> <code>list</code> <code>None</code> <code>uploader</code> <code>list</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>study</code> <code>int</code> <code>None</code> <code>display_errors</code> <code>(bool, optional(default=None))</code> <p>Whether to list runs which have an error (for example a missing prediction file).</p> <code>False</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>dict</code> <p>Legal filter operators: task_type.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict of dicts, or dataframe</code> Source code in <code>openml/runs/functions.py</code> <pre><code>def list_runs(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    id: list | None = None,  # noqa: A002\n    task: list[int] | None = None,\n    setup: list | None = None,\n    flow: list | None = None,\n    uploader: list | None = None,\n    tag: str | None = None,\n    study: int | None = None,\n    display_errors: bool = False,  # noqa: FBT001, FBT002\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all runs matching all of the given filters.\n    (Supports large amount of results)\n\n    Parameters\n    ----------\n    offset : int, optional\n        the number of runs to skip, starting from the first\n    size : int, optional\n        the maximum number of runs to show\n\n    id : list, optional\n\n    task : list, optional\n\n    setup: list, optional\n\n    flow : list, optional\n\n    uploader : list, optional\n\n    tag : str, optional\n\n    study : int, optional\n\n    display_errors : bool, optional (default=None)\n        Whether to list runs which have an error (for example a missing\n        prediction file).\n\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    kwargs : dict, optional\n        Legal filter operators: task_type.\n\n    Returns\n    -------\n    dict of dicts, or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\"Invalid output format selected. Only 'dict' or 'dataframe' applicable.\")\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    # TODO(eddiebergman): Do we really need this runtime type validation?\n    if id is not None and (not isinstance(id, list)):\n        raise TypeError(\"id must be of type list.\")\n    if task is not None and (not isinstance(task, list)):\n        raise TypeError(\"task must be of type list.\")\n    if setup is not None and (not isinstance(setup, list)):\n        raise TypeError(\"setup must be of type list.\")\n    if flow is not None and (not isinstance(flow, list)):\n        raise TypeError(\"flow must be of type list.\")\n    if uploader is not None and (not isinstance(uploader, list)):\n        raise TypeError(\"uploader must be of type list.\")\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_runs,\n        offset=offset,\n        size=size,\n        id=id,\n        task=task,\n        setup=setup,\n        flow=flow,\n        uploader=uploader,\n        tag=tag,\n        study=study,\n        display_errors=display_errors,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.run_exists","title":"<code>run_exists(task_id, setup_id)</code>","text":"<p>Checks whether a task/setup combination is already present on the server.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> required <code>setup_id</code> <code>int</code> required <p>Returns:</p> Type Description <code>    Set run ids for runs where flow setup_id was run on task_id. Empty</code> <p>set if it wasn't run yet.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_exists(task_id: int, setup_id: int) -&gt; set[int]:\n    \"\"\"Checks whether a task/setup combination is already present on the\n    server.\n\n    Parameters\n    ----------\n    task_id : int\n\n    setup_id : int\n\n    Returns\n    -------\n        Set run ids for runs where flow setup_id was run on task_id. Empty\n        set if it wasn't run yet.\n    \"\"\"\n    if setup_id &lt;= 0:\n        # openml setups are in range 1-inf\n        return set()\n\n    try:\n        result = list_runs(task=[task_id], setup=[setup_id], output_format=\"dataframe\")\n        assert isinstance(result, pd.DataFrame)  # TODO(eddiebergman): Remove once #1299\n        return set() if result.empty else set(result[\"run_id\"])\n    except OpenMLServerException as exception:\n        # error code implies no results. The run does not exist yet\n        if exception.code != ERROR_CODE:\n            raise exception\n        return set()\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.run_flow_on_task","title":"<code>run_flow_on_task(flow, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model provided by the flow on the dataset defined by task.</p> <p>Takes the flow and repeat information into account. The Flow may optionally be published.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>A flow wraps a machine learning model together with relevant information. The model has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask</code> <p>Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_flow_on_task(  # noqa: C901, PLR0912, PLR0915, PLR0913\n    flow: OpenMLFlow,\n    task: OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT002, FBT001\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun:\n    \"\"\"Run the model provided by the flow on the dataset defined by task.\n\n    Takes the flow and repeat information into account.\n    The Flow may optionally be published.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        A flow wraps a machine learning model together with relevant information.\n        The model has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask\n        Task to perform. This may be an OpenMLFlow instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    \"\"\"\n    if flow_tags is not None and not isinstance(flow_tags, list):\n        raise ValueError(\"flow_tags should be a list\")\n\n    # TODO: At some point in the future do not allow for arguments in old order (changed 6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    if isinstance(flow, OpenMLTask) and isinstance(task, OpenMLFlow):\n        # We want to allow either order of argument (to avoid confusion).\n        warnings.warn(\n            \"The old argument order (Flow, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, Flow).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, flow = flow, task\n\n    if task.task_id is None:\n        raise ValueError(\"The task should be published at OpenML\")\n\n    if flow.model is None:\n        flow.model = flow.extension.flow_to_model(flow)\n\n    flow.model = flow.extension.seed_model(flow.model, seed=seed)\n\n    # We only need to sync with the server right now if we want to upload the flow,\n    # or ensure no duplicate runs exist. Otherwise it can be synced at upload time.\n    flow_id = None\n    if upload_flow or avoid_duplicate_runs:\n        flow_id = flow_exists(flow.name, flow.external_version)\n        if isinstance(flow.flow_id, int) and flow_id != flow.flow_id:\n            if flow_id is not False:\n                raise PyOpenMLError(\n                    \"Local flow_id does not match server flow_id: \"\n                    f\"'{flow.flow_id}' vs '{flow_id}'\",\n                )\n            raise PyOpenMLError(\n                \"Flow does not exist on the server, but 'flow.flow_id' is not None.\"\n            )\n        if upload_flow and flow_id is False:\n            flow.publish()\n            flow_id = flow.flow_id\n        elif flow_id:\n            flow_from_server = get_flow(flow_id)\n            _copy_server_fields(flow_from_server, flow)\n            if avoid_duplicate_runs:\n                flow_from_server.model = flow.model\n                setup_id = setup_exists(flow_from_server)\n                ids = run_exists(task.task_id, setup_id)\n                if ids:\n                    error_message = (\n                        \"One or more runs of this setup were already performed on the task.\"\n                    )\n                    raise OpenMLRunsExistError(ids, error_message)\n        else:\n            # Flow does not exist on server and we do not want to upload it.\n            # No sync with the server happens.\n            flow_id = None\n\n    dataset = task.get_dataset()\n\n    run_environment = flow.extension.get_version_information()\n    tags = [\"openml-python\", run_environment[1]]\n\n    if flow.extension.check_if_model_fitted(flow.model):\n        warnings.warn(\n            \"The model is already fitted!\"\n            \" This might cause inconsistency in comparison of results.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # execute the run\n    res = _run_task_get_arffcontent(\n        model=flow.model,\n        task=task,\n        extension=flow.extension,\n        add_local_measures=add_local_measures,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n\n    data_content, trace, fold_evaluations, sample_evaluations = res\n    fields = [*run_environment, time.strftime(\"%c\"), \"Created by run_flow_on_task\"]\n    generated_description = \"\\n\".join(fields)\n    run = OpenMLRun(\n        task_id=task.task_id,\n        flow_id=flow_id,\n        dataset_id=dataset.dataset_id,\n        model=flow.model,\n        flow_name=flow.name,\n        tags=tags,\n        trace=trace,\n        data_content=data_content,\n        flow=flow,\n        setup_string=flow.extension.create_setup_string(flow.model),\n        description_text=generated_description,\n    )\n\n    if (upload_flow or avoid_duplicate_runs) and flow.flow_id is not None:\n        # We only extract the parameter settings if a sync happened with the server.\n        # I.e. when the flow was uploaded or we found it in the avoid_duplicate check.\n        # Otherwise, we will do this at upload time.\n        run.parameter_settings = flow.extension.obtain_parameter_values(flow)\n\n    # now we need to attach the detailed evaluations\n    if task.task_type_id == TaskType.LEARNING_CURVE:\n        run.sample_evaluations = sample_evaluations\n    else:\n        run.fold_evaluations = fold_evaluations\n\n    if flow_id:\n        message = f\"Executed Task {task.task_id} with Flow id:{run.flow_id}\"\n    else:\n        message = f\"Executed Task {task.task_id} on local Flow with name {flow.name}.\"\n    config.logger.info(message)\n\n    return run\n</code></pre>"},{"location":"reference/runs/functions/#openml.runs.functions.run_model_on_task","title":"<code>run_model_on_task(model, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True, upload_flow=False, return_flow=False, dataset_format='dataframe', n_jobs=None)</code>","text":"<p>Run the model on the dataset defined by the task.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>sklearn model</code> <p>A model which has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)</p> required <code>task</code> <code>OpenMLTask or int or str</code> <p>Task to perform or Task id. This may be a model instead if the first argument is an OpenMLTask.</p> required <code>avoid_duplicate_runs</code> <code>(bool, optional(default=True))</code> <p>If True, the run will throw an error if the setup/task combination is already present on the server. This feature requires an internet connection.</p> <code>True</code> <code>flow_tags</code> <code>(List[str], optional(default=None))</code> <p>A list of tags that the flow should have at creation.</p> <code>None</code> <code>seed</code> <code>int | None</code> <p>Models that are not seeded will get this seed.</p> <code>None</code> <code>add_local_measures</code> <code>(bool, optional(default=True))</code> <p>Determines whether to calculate a set of evaluation measures locally, to later verify server behaviour.</p> <code>True</code> <code>upload_flow</code> <code>bool(default=False)</code> <p>If True, upload the flow to OpenML if it does not exist yet. If False, do not upload the flow to OpenML.</p> <code>False</code> <code>return_flow</code> <code>bool(default=False)</code> <p>If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.</p> <code>False</code> <code>dataset_format</code> <code>str(default='dataframe')</code> <p>If 'array', the dataset is passed to the model as a numpy array. If 'dataframe', the dataset is passed to the model as a pandas dataframe.</p> <code>'dataframe'</code> <code>n_jobs</code> <code>int(default=None)</code> <p>The number of processes/threads to distribute the evaluation asynchronously. If <code>None</code> or <code>1</code>, then the evaluation is treated as synchronous and processed sequentially. If <code>-1</code>, then the job uses as many cores available.</p> <code>None</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>Result of the run.</p> <code>flow</code> <code>OpenMLFlow (optional, only if `return_flow` is True).</code> <p>Flow generated from the model.</p> Source code in <code>openml/runs/functions.py</code> <pre><code>def run_model_on_task(  # noqa: PLR0913\n    model: Any,\n    task: int | str | OpenMLTask,\n    avoid_duplicate_runs: bool = True,  # noqa: FBT001, FBT002\n    flow_tags: list[str] | None = None,\n    seed: int | None = None,\n    add_local_measures: bool = True,  # noqa: FBT001, FBT002\n    upload_flow: bool = False,  # noqa: FBT001, FBT002\n    return_flow: bool = False,  # noqa: FBT001, FBT002\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"dataframe\",\n    n_jobs: int | None = None,\n) -&gt; OpenMLRun | tuple[OpenMLRun, OpenMLFlow]:\n    \"\"\"Run the model on the dataset defined by the task.\n\n    Parameters\n    ----------\n    model : sklearn model\n        A model which has a function fit(X,Y) and predict(X),\n        all supervised estimators of scikit learn follow this definition of a model\n        (https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html)\n    task : OpenMLTask or int or str\n        Task to perform or Task id.\n        This may be a model instead if the first argument is an OpenMLTask.\n    avoid_duplicate_runs : bool, optional (default=True)\n        If True, the run will throw an error if the setup/task combination is already present on\n        the server. This feature requires an internet connection.\n    flow_tags : List[str], optional (default=None)\n        A list of tags that the flow should have at creation.\n    seed: int, optional (default=None)\n        Models that are not seeded will get this seed.\n    add_local_measures : bool, optional (default=True)\n        Determines whether to calculate a set of evaluation measures locally,\n        to later verify server behaviour.\n    upload_flow : bool (default=False)\n        If True, upload the flow to OpenML if it does not exist yet.\n        If False, do not upload the flow to OpenML.\n    return_flow : bool (default=False)\n        If True, returns the OpenMLFlow generated from the model in addition to the OpenMLRun.\n    dataset_format : str (default='dataframe')\n        If 'array', the dataset is passed to the model as a numpy array.\n        If 'dataframe', the dataset is passed to the model as a pandas dataframe.\n    n_jobs : int (default=None)\n        The number of processes/threads to distribute the evaluation asynchronously.\n        If `None` or `1`, then the evaluation is treated as synchronous and processed sequentially.\n        If `-1`, then the job uses as many cores available.\n\n    Returns\n    -------\n    run : OpenMLRun\n        Result of the run.\n    flow : OpenMLFlow (optional, only if `return_flow` is True).\n        Flow generated from the model.\n    \"\"\"\n    if avoid_duplicate_runs and not config.apikey:\n        warnings.warn(\n            \"avoid_duplicate_runs is set to True, but no API key is set. \"\n            \"Please set your API key in the OpenML configuration file, see\"\n            \"https://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial\"\n            \".html#authentication for more information on authentication.\",\n            RuntimeWarning,\n            stacklevel=2,\n        )\n\n    # TODO: At some point in the future do not allow for arguments in old order (6-2018).\n    # Flexibility currently still allowed due to code-snippet in OpenML100 paper (3-2019).\n    # When removing this please also remove the method `is_estimator` from the extension\n    # interface as it is only used here (MF, 3-2019)\n    if isinstance(model, (int, str, OpenMLTask)):\n        warnings.warn(\n            \"The old argument order (task, model) is deprecated and \"\n            \"will not be supported in the future. Please use the \"\n            \"order (model, task).\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        task, model = model, task\n\n    extension = get_extension_by_model(model, raise_if_no_extension=True)\n    if extension is None:\n        # This should never happen and is only here to please mypy will be gone soon once the\n        # whole function is removed\n        raise TypeError(extension)\n\n    flow = extension.model_to_flow(model)\n\n    def get_task_and_type_conversion(_task: int | str | OpenMLTask) -&gt; OpenMLTask:\n        \"\"\"Retrieve an OpenMLTask object from either an integer or string ID,\n        or directly from an OpenMLTask object.\n\n        Parameters\n        ----------\n        _task : Union[int, str, OpenMLTask]\n            The task ID or the OpenMLTask object.\n\n        Returns\n        -------\n        OpenMLTask\n            The OpenMLTask object.\n        \"\"\"\n        if isinstance(_task, (int, str)):\n            return get_task(int(_task))  # type: ignore\n\n        return _task\n\n    task = get_task_and_type_conversion(task)\n\n    run = run_flow_on_task(\n        task=task,\n        flow=flow,\n        avoid_duplicate_runs=avoid_duplicate_runs,\n        flow_tags=flow_tags,\n        seed=seed,\n        add_local_measures=add_local_measures,\n        upload_flow=upload_flow,\n        dataset_format=dataset_format,\n        n_jobs=n_jobs,\n    )\n    if return_flow:\n        return run, flow\n    return run\n</code></pre>"},{"location":"reference/runs/run/","title":"run","text":""},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun","title":"<code>OpenMLRun</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Run: result of running a model on an OpenML dataset.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The ID of the OpenML task associated with the run.</p> required <code>flow_id</code> <code>int | None</code> <p>The ID of the OpenML flow associated with the run.</p> required <code>dataset_id</code> <code>int | None</code> <p>The ID of the OpenML dataset used for the run.</p> required <code>setup_string</code> <code>str | None</code> <p>The setup string of the run.</p> <code>None</code> <code>output_files</code> <code>dict[str, int] | None</code> <p>Specifies where each related file can be found.</p> <code>None</code> <code>setup_id</code> <code>int | None</code> <p>An integer representing the ID of the setup used for the run.</p> <code>None</code> <code>tags</code> <code>list[str] | None</code> <p>Representing the tags associated with the run.</p> <code>None</code> <code>uploader</code> <code>int | None</code> <p>User ID of the uploader.</p> <code>None</code> <code>uploader_name</code> <code>str | None</code> <p>The name of the person who uploaded the run.</p> <code>None</code> <code>evaluations</code> <code>dict | None</code> <p>Representing the evaluations of the run.</p> <code>None</code> <code>fold_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each fold.</p> <code>None</code> <code>sample_evaluations</code> <code>dict | None</code> <p>The evaluations of the run for each sample.</p> <code>None</code> <code>data_content</code> <code>list[list] | None</code> <p>The predictions generated from executing this run.</p> <code>None</code> <code>trace</code> <code>OpenMLRunTrace | None</code> <p>The trace containing information on internal model evaluations of this run.</p> <code>None</code> <code>model</code> <code>object | None</code> <p>The untrained model that was evaluated in the run.</p> <code>None</code> <code>task_type</code> <code>str | None</code> <p>The type of the OpenML task associated with the run.</p> <code>None</code> <code>task_evaluation_measure</code> <code>str | None</code> <p>The evaluation measure used for the task.</p> <code>None</code> <code>flow_name</code> <code>str | None</code> <p>The name of the OpenML flow associated with the run.</p> <code>None</code> <code>parameter_settings</code> <code>list[dict[str, Any]] | None</code> <p>Representing the parameter settings used for the run.</p> <code>None</code> <code>predictions_url</code> <code>str | None</code> <p>The URL of the predictions file.</p> <code>None</code> <code>task</code> <code>OpenMLTask | None</code> <p>An instance of the OpenMLTask class, representing the OpenML task associated with the run.</p> <code>None</code> <code>flow</code> <code>OpenMLFlow | None</code> <p>An instance of the OpenMLFlow class, representing the OpenML flow associated with the run.</p> <code>None</code> <code>run_id</code> <code>int | None</code> <p>The ID of the run.</p> <code>None</code> <code>description_text</code> <code>str | None</code> <p>Description text to add to the predictions file. If left None, is set to the time the arff file is generated.</p> <code>None</code> <code>run_details</code> <code>str | None</code> <p>Description of the run stored in the run meta-data.</p> <code>None</code> Source code in <code>openml/runs/run.py</code> <pre><code>class OpenMLRun(OpenMLBase):\n    \"\"\"OpenML Run: result of running a model on an OpenML dataset.\n\n    Parameters\n    ----------\n    task_id: int\n        The ID of the OpenML task associated with the run.\n    flow_id: int\n        The ID of the OpenML flow associated with the run.\n    dataset_id: int\n        The ID of the OpenML dataset used for the run.\n    setup_string: str\n        The setup string of the run.\n    output_files: Dict[str, int]\n        Specifies where each related file can be found.\n    setup_id: int\n        An integer representing the ID of the setup used for the run.\n    tags: List[str]\n        Representing the tags associated with the run.\n    uploader: int\n        User ID of the uploader.\n    uploader_name: str\n        The name of the person who uploaded the run.\n    evaluations: Dict\n        Representing the evaluations of the run.\n    fold_evaluations: Dict\n        The evaluations of the run for each fold.\n    sample_evaluations: Dict\n        The evaluations of the run for each sample.\n    data_content: List[List]\n        The predictions generated from executing this run.\n    trace: OpenMLRunTrace\n        The trace containing information on internal model evaluations of this run.\n    model: object\n        The untrained model that was evaluated in the run.\n    task_type: str\n        The type of the OpenML task associated with the run.\n    task_evaluation_measure: str\n        The evaluation measure used for the task.\n    flow_name: str\n        The name of the OpenML flow associated with the run.\n    parameter_settings: list[OrderedDict]\n        Representing the parameter settings used for the run.\n    predictions_url: str\n        The URL of the predictions file.\n    task: OpenMLTask\n        An instance of the OpenMLTask class, representing the OpenML task associated\n        with the run.\n    flow: OpenMLFlow\n        An instance of the OpenMLFlow class, representing the OpenML flow associated\n        with the run.\n    run_id: int\n        The ID of the run.\n    description_text: str, optional\n        Description text to add to the predictions file. If left None, is set to the\n        time the arff file is generated.\n    run_details: str, optional (default=None)\n        Description of the run stored in the run meta-data.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int,\n        flow_id: int | None,\n        dataset_id: int | None,\n        setup_string: str | None = None,\n        output_files: dict[str, int] | None = None,\n        setup_id: int | None = None,\n        tags: list[str] | None = None,\n        uploader: int | None = None,\n        uploader_name: str | None = None,\n        evaluations: dict | None = None,\n        fold_evaluations: dict | None = None,\n        sample_evaluations: dict | None = None,\n        data_content: list[list] | None = None,\n        trace: OpenMLRunTrace | None = None,\n        model: object | None = None,\n        task_type: str | None = None,\n        task_evaluation_measure: str | None = None,\n        flow_name: str | None = None,\n        parameter_settings: list[dict[str, Any]] | None = None,\n        predictions_url: str | None = None,\n        task: OpenMLTask | None = None,\n        flow: OpenMLFlow | None = None,\n        run_id: int | None = None,\n        description_text: str | None = None,\n        run_details: str | None = None,\n    ):\n        self.uploader = uploader\n        self.uploader_name = uploader_name\n        self.task_id = task_id\n        self.task_type = task_type\n        self.task_evaluation_measure = task_evaluation_measure\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.setup_id = setup_id\n        self.setup_string = setup_string\n        self.parameter_settings = parameter_settings\n        self.dataset_id = dataset_id\n        self.evaluations = evaluations\n        self.fold_evaluations = fold_evaluations\n        self.sample_evaluations = sample_evaluations\n        self.data_content = data_content\n        self.output_files = output_files\n        self.trace = trace\n        self.error_message = None\n        self.task = task\n        self.flow = flow\n        self.run_id = run_id\n        self.model = model\n        self.tags = tags\n        self.predictions_url = predictions_url\n        self.description_text = description_text\n        self.run_details = run_details\n        self._predictions = None\n\n    @property\n    def predictions(self) -&gt; pd.DataFrame:\n        \"\"\"Return a DataFrame with predictions for this run\"\"\"\n        if self._predictions is None:\n            if self.data_content:\n                arff_dict = self._generate_arff_dict()\n            elif self.predictions_url:\n                arff_text = openml._api_calls._download_text_file(self.predictions_url)\n                arff_dict = arff.loads(arff_text)\n            else:\n                raise RuntimeError(\"Run has no predictions.\")\n            self._predictions = pd.DataFrame(\n                arff_dict[\"data\"],\n                columns=[name for name, _ in arff_dict[\"attributes\"]],\n            )\n        return self._predictions\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"The ID of the run, None if not uploaded to the server yet.\"\"\"\n        return self.run_id\n\n    def _evaluation_summary(self, metric: str) -&gt; str:\n        \"\"\"Summarizes the evaluation of a metric over all folds.\n\n        The fold scores for the metric must exist already. During run creation,\n        by default, the MAE for OpenMLRegressionTask and the accuracy for\n        OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed.\n\n        If repetition exist, we take the mean over all repetitions.\n\n        Parameters\n        ----------\n        metric: str\n            Name of an evaluation metric that was used to compute fold scores.\n\n        Returns\n        -------\n        metric_summary: str\n            A formatted string that displays the metric's evaluation summary.\n            The summary consists of the mean and std.\n        \"\"\"\n        if self.fold_evaluations is None:\n            raise ValueError(\"No fold evaluations available.\")\n        fold_score_lists = self.fold_evaluations[metric].values()\n\n        # Get the mean and std over all repetitions\n        rep_means = [np.mean(list(x.values())) for x in fold_score_lists]\n        rep_stds = [np.std(list(x.values())) for x in fold_score_lists]\n\n        return f\"{np.mean(rep_means):.4f} +- {np.mean(rep_stds):.4f}\"\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        # Set up fields\n        fields = {\n            \"Uploader Name\": self.uploader_name,\n            \"Metric\": self.task_evaluation_measure,\n            \"Run ID\": self.run_id,\n            \"Task ID\": self.task_id,\n            \"Task Type\": self.task_type,\n            \"Task URL\": openml.tasks.OpenMLTask.url_for_id(self.task_id),\n            \"Flow ID\": self.flow_id,\n            \"Flow Name\": self.flow_name,\n            \"Flow URL\": (\n                openml.flows.OpenMLFlow.url_for_id(self.flow_id)\n                if self.flow_id is not None\n                else None\n            ),\n            \"Setup ID\": self.setup_id,\n            \"Setup String\": self.setup_string,\n            \"Dataset ID\": self.dataset_id,\n            \"Dataset URL\": (\n                openml.datasets.OpenMLDataset.url_for_id(self.dataset_id)\n                if self.dataset_id is not None\n                else None\n            ),\n        }\n\n        # determines the order of the initial fields in which the information will be printed\n        order = [\"Uploader Name\", \"Uploader Profile\", \"Metric\", \"Result\"]\n\n        if self.uploader is not None:\n            fields[\"Uploader Profile\"] = f\"{openml.config.get_server_base_url()}/u/{self.uploader}\"\n        if self.run_id is not None:\n            fields[\"Run URL\"] = self.openml_url\n        if self.evaluations is not None and self.task_evaluation_measure in self.evaluations:\n            fields[\"Result\"] = self.evaluations[self.task_evaluation_measure]\n        elif self.fold_evaluations is not None:\n            # -- Add locally computed summary values if possible\n            if \"predictive_accuracy\" in self.fold_evaluations:\n                # OpenMLClassificationTask; OpenMLLearningCurveTask\n                result_field = \"Local Result - Accuracy (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"predictive_accuracy\")\n                order.append(result_field)\n            elif \"mean_absolute_error\" in self.fold_evaluations:\n                # OpenMLRegressionTask\n                result_field = \"Local Result - MAE (+- STD)\"\n                fields[result_field] = self._evaluation_summary(\"mean_absolute_error\")\n                order.append(result_field)\n\n            if \"usercpu_time_millis\" in self.fold_evaluations:\n                # Runtime should be available for most tasks types\n                rt_field = \"Local Runtime - ms (+- STD)\"\n                fields[rt_field] = self._evaluation_summary(\"usercpu_time_millis\")\n                order.append(rt_field)\n\n        # determines the remaining order\n        order += [\n            \"Run ID\",\n            \"Run URL\",\n            \"Task ID\",\n            \"Task Type\",\n            \"Task URL\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Setup ID\",\n            \"Setup String\",\n            \"Dataset ID\",\n            \"Dataset URL\",\n        ]\n        return [\n            (key, \"None\" if fields[key] is None else fields[key])  # type: ignore\n            for key in order\n            if key in fields\n        ]\n\n    @classmethod\n    def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n        \"\"\"\n        The inverse of the to_filesystem method. Instantiates an OpenMLRun\n        object based on files stored on the file system.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            are stored\n\n        expect_model : bool\n            if True, it requires the model pickle to be present, and an error\n            will be thrown if not. Otherwise, the model might or might not\n            be present.\n\n        Returns\n        -------\n        run : OpenMLRun\n            the re-instantiated run object\n        \"\"\"\n        # Avoiding cyclic imports\n        import openml.runs.functions\n\n        directory = Path(directory)\n        if not directory.is_dir():\n            raise ValueError(\"Could not find folder\")\n\n        description_path = directory / \"description.xml\"\n        predictions_path = directory / \"predictions.arff\"\n        trace_path = directory / \"trace.arff\"\n        model_path = directory / \"model.pkl\"\n\n        if not description_path.is_file():\n            raise ValueError(\"Could not find description.xml\")\n        if not predictions_path.is_file():\n            raise ValueError(\"Could not find predictions.arff\")\n        if (not model_path.is_file()) and expect_model:\n            raise ValueError(\"Could not find model.pkl\")\n\n        with description_path.open() as fht:\n            xml_string = fht.read()\n        run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n        if run.flow_id is None:\n            flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n            run.flow = flow\n            run.flow_name = flow.name\n\n        with predictions_path.open() as fht:\n            predictions = arff.load(fht)\n            run.data_content = predictions[\"data\"]\n\n        if model_path.is_file():\n            # note that it will load the model if the file exists, even if\n            # expect_model is False\n            with model_path.open(\"rb\") as fhb:\n                run.model = pickle.load(fhb)  # noqa: S301\n\n        if trace_path.is_file():\n            run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n        return run\n\n    def to_filesystem(\n        self,\n        directory: str | Path,\n        store_model: bool = True,  # noqa: FBT001, FBT002\n    ) -&gt; None:\n        \"\"\"\n        The inverse of the from_filesystem method. Serializes a run\n        on the filesystem, to be uploaded later.\n\n        Parameters\n        ----------\n        directory : str\n            a path leading to the folder where the results\n            will be stored. Should be empty\n\n        store_model : bool, optional (default=True)\n            if True, a model will be pickled as well. As this is the most\n            storage expensive part, it is often desirable to not store the\n            model.\n        \"\"\"\n        if self.data_content is None or self.model is None:\n            raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n        directory = Path(directory)\n        directory.mkdir(exist_ok=True, parents=True)\n\n        if any(directory.iterdir()):\n            raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n        run_xml = self._to_xml()\n        predictions_arff = arff.dumps(self._generate_arff_dict())\n\n        # It seems like typing does not allow to define the same variable multiple times\n        with (directory / \"description.xml\").open(\"w\") as fh:\n            fh.write(run_xml)\n        with (directory / \"predictions.arff\").open(\"w\") as fh:\n            fh.write(predictions_arff)\n        if store_model:\n            with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n                pickle.dump(self.model, fh_b)\n\n        if self.flow_id is None and self.flow is not None:\n            self.flow.to_filesystem(directory)\n\n        if self.trace is not None:\n            self.trace._to_filesystem(directory)\n\n    def _generate_arff_dict(self) -&gt; OrderedDict[str, Any]:\n        \"\"\"Generates the arff dictionary for uploading predictions to the\n        server.\n\n        Assumes that the run has been executed.\n\n        The order of the attributes follows the order defined by the Client API for R.\n\n        Returns\n        -------\n        arf_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains predictions and information about the run environment.\n        \"\"\"\n        if self.data_content is None:\n            raise ValueError(\"Run has not been executed.\")\n        if self.flow is None:\n            assert self.flow_id is not None, \"Run has no associated flow id!\"\n            self.flow = get_flow(self.flow_id)\n\n        if self.description_text is None:\n            self.description_text = time.strftime(\"%c\")\n        task = get_task(self.task_id)\n\n        arff_dict = OrderedDict()  # type: 'OrderedDict[str, Any]'\n        arff_dict[\"data\"] = self.data_content\n        arff_dict[\"description\"] = self.description_text\n        arff_dict[\"relation\"] = f\"openml_task_{task.task_id}_predictions\"\n\n        if isinstance(task, OpenMLLearningCurveTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"]\n                    + [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                    + [\n                        (\"confidence.\" + class_labels[i], \"NUMERIC\")\n                        for i in range(len(class_labels))\n                    ]\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLClassificationTask):\n            class_labels = task.class_labels\n            instance_specifications = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"sample\", \"NUMERIC\"),  # Legacy\n                (\"row_id\", \"NUMERIC\"),\n            ]\n\n            arff_dict[\"attributes\"] = instance_specifications\n            if class_labels is not None:\n                prediction_confidences = [\n                    (\"confidence.\" + class_labels[i], \"NUMERIC\") for i in range(len(class_labels))\n                ]\n                prediction_and_true = [(\"prediction\", class_labels), (\"correct\", class_labels)]\n                arff_dict[\"attributes\"] = (\n                    arff_dict[\"attributes\"] + prediction_and_true + prediction_confidences\n                )\n            else:\n                raise ValueError(\"The task has no class labels\")\n\n        elif isinstance(task, OpenMLRegressionTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"prediction\", \"NUMERIC\"),\n                (\"truth\", \"NUMERIC\"),\n            ]\n\n        elif isinstance(task, OpenMLClusteringTask):\n            arff_dict[\"attributes\"] = [\n                (\"repeat\", \"NUMERIC\"),\n                (\"fold\", \"NUMERIC\"),\n                (\"row_id\", \"NUMERIC\"),\n                (\"cluster\", \"NUMERIC\"),\n            ]\n\n        else:\n            raise NotImplementedError(\"Task type %s is not yet supported.\" % str(task.task_type))\n\n        return arff_dict\n\n    def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n        \"\"\"Calculates metric scores based on predicted values. Assumes the\n        run has been executed locally (and contains run_data). Furthermore,\n        it assumes that the 'correct' or 'truth' attribute is specified in\n        the arff (which is an optional field, but always the case for\n        openml-python runs)\n\n        Parameters\n        ----------\n        sklearn_fn : function\n            a function pointer to a sklearn function that\n            accepts ``y_true``, ``y_pred`` and ``**kwargs``\n        kwargs : dict\n            kwargs for the function\n\n        Returns\n        -------\n        scores : ndarray of scores of length num_folds * num_repeats\n            metric results\n        \"\"\"\n        kwargs = kwargs if kwargs else {}\n        if self.data_content is not None and self.task_id is not None:\n            predictions_arff = self._generate_arff_dict()\n        elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n            predictions_file_url = openml._api_calls._file_id_to_url(\n                self.output_files[\"predictions\"],\n                \"predictions.arff\",\n            )\n            response = openml._api_calls._download_text_file(predictions_file_url)\n            predictions_arff = arff.loads(response)\n            # TODO: make this a stream reader\n        else:\n            raise ValueError(\n                \"Run should have been locally executed or \" \"contain outputfile reference.\",\n            )\n\n        # Need to know more about the task to compute scores correctly\n        task = get_task(self.task_id)\n\n        attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n        if (\n            task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n            and \"correct\" not in attribute_names\n        ):\n            raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n        if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n            raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n        if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n            raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n        def _attribute_list_to_dict(attribute_list):  # type: ignore\n            # convenience function: Creates a mapping to map from the name of\n            # attributes present in the arff prediction file to their index.\n            # This is necessary because the number of classes can be different\n            # for different tasks.\n            res = OrderedDict()\n            for idx in range(len(attribute_list)):\n                res[attribute_list[idx][0]] = idx\n            return res\n\n        attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n        repeat_idx = attribute_dict[\"repeat\"]\n        fold_idx = attribute_dict[\"fold\"]\n        predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n        if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n            correct_idx = attribute_dict[\"correct\"]\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            correct_idx = attribute_dict[\"truth\"]\n        has_samples = False\n        if \"sample\" in attribute_dict:\n            sample_idx = attribute_dict[\"sample\"]\n            has_samples = True\n\n        if (\n            predictions_arff[\"attributes\"][predicted_idx][1]\n            != predictions_arff[\"attributes\"][correct_idx][1]\n        ):\n            pred = predictions_arff[\"attributes\"][predicted_idx][1]\n            corr = predictions_arff[\"attributes\"][correct_idx][1]\n            raise ValueError(\n                \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n            )\n\n        # TODO: these could be cached\n        values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n        values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n        for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n            rep = line[repeat_idx]\n            fold = line[fold_idx]\n            samp = line[sample_idx] if has_samples else 0\n\n            if task.task_type_id in [\n                TaskType.SUPERVISED_CLASSIFICATION,\n                TaskType.LEARNING_CURVE,\n            ]:\n                prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                    line[predicted_idx],\n                )\n                correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n            elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n                prediction = line[predicted_idx]\n                correct = line[correct_idx]\n            if rep not in values_predict:\n                values_predict[rep] = OrderedDict()\n                values_correct[rep] = OrderedDict()\n            if fold not in values_predict[rep]:\n                values_predict[rep][fold] = OrderedDict()\n                values_correct[rep][fold] = OrderedDict()\n            if samp not in values_predict[rep][fold]:\n                values_predict[rep][fold][samp] = []\n                values_correct[rep][fold][samp] = []\n\n            values_predict[rep][fold][samp].append(prediction)\n            values_correct[rep][fold][samp].append(correct)\n\n        scores = []\n        for rep in values_predict:\n            for fold in values_predict[rep]:\n                last_sample = len(values_predict[rep][fold]) - 1\n                y_pred = values_predict[rep][fold][last_sample]\n                y_true = values_correct[rep][fold][last_sample]\n                scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n        return np.array(scores)\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.run_id = int(xml_response[\"oml:upload_run\"][\"oml:run_id\"])\n\n    def _get_file_elements(self) -&gt; dict:\n        \"\"\"Get file_elements to upload to the server.\n\n        Derived child classes should overwrite this method as necessary.\n        The description field will be populated automatically if not provided.\n        \"\"\"\n        if self.parameter_settings is None and self.model is None:\n            raise PyOpenMLError(\n                \"OpenMLRun must contain a model or be initialized with parameter_settings.\",\n            )\n        if self.flow_id is None:\n            if self.flow is None:\n                raise PyOpenMLError(\n                    \"OpenMLRun object does not contain a flow id or reference to OpenMLFlow \"\n                    \"(these should have been added while executing the task). \",\n                )\n\n            # publish the linked Flow before publishing the run.\n            self.flow.publish()\n            self.flow_id = self.flow.flow_id\n\n        if self.parameter_settings is None:\n            if self.flow is None:\n                assert self.flow_id is not None  # for mypy\n                self.flow = openml.flows.get_flow(self.flow_id)\n            self.parameter_settings = self.flow.extension.obtain_parameter_values(\n                self.flow,\n                self.model,\n            )\n\n        file_elements = {\"description\": (\"description.xml\", self._to_xml())}\n\n        if self.error_message is None:\n            predictions = arff.dumps(self._generate_arff_dict())\n            file_elements[\"predictions\"] = (\"predictions.arff\", predictions)\n\n        if self.trace is not None:\n            trace_arff = arff.dumps(self.trace.trace_to_arff())\n            file_elements[\"trace\"] = (\"trace.arff\", trace_arff)\n        return file_elements\n\n    def _to_dict(self) -&gt; dict[str, dict]:  # noqa: PLR0912, C901\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        description = OrderedDict()  # type: 'OrderedDict'\n        description[\"oml:run\"] = OrderedDict()\n        description[\"oml:run\"][\"@xmlns:oml\"] = \"http://openml.org/openml\"\n        description[\"oml:run\"][\"oml:task_id\"] = self.task_id\n        description[\"oml:run\"][\"oml:flow_id\"] = self.flow_id\n        if self.setup_string is not None:\n            description[\"oml:run\"][\"oml:setup_string\"] = self.setup_string\n        if self.error_message is not None:\n            description[\"oml:run\"][\"oml:error_message\"] = self.error_message\n        if self.run_details is not None:\n            description[\"oml:run\"][\"oml:run_details\"] = self.run_details\n        description[\"oml:run\"][\"oml:parameter_setting\"] = self.parameter_settings\n        if self.tags is not None:\n            description[\"oml:run\"][\"oml:tag\"] = self.tags\n        if (self.fold_evaluations is not None and len(self.fold_evaluations) &gt; 0) or (\n            self.sample_evaluations is not None and len(self.sample_evaluations) &gt; 0\n        ):\n            description[\"oml:run\"][\"oml:output_data\"] = OrderedDict()\n            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"] = []\n        if self.fold_evaluations is not None:\n            for measure in self.fold_evaluations:\n                for repeat in self.fold_evaluations[measure]:\n                    for fold, value in self.fold_evaluations[measure][repeat].items():\n                        current = OrderedDict(\n                            [\n                                (\"@repeat\", str(repeat)),\n                                (\"@fold\", str(fold)),\n                                (\"oml:name\", measure),\n                                (\"oml:value\", str(value)),\n                            ],\n                        )\n                        description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(current)\n        if self.sample_evaluations is not None:\n            for measure in self.sample_evaluations:\n                for repeat in self.sample_evaluations[measure]:\n                    for fold in self.sample_evaluations[measure][repeat]:\n                        for sample, value in self.sample_evaluations[measure][repeat][fold].items():\n                            current = OrderedDict(\n                                [\n                                    (\"@repeat\", str(repeat)),\n                                    (\"@fold\", str(fold)),\n                                    (\"@sample\", str(sample)),\n                                    (\"oml:name\", measure),\n                                    (\"oml:value\", str(value)),\n                                ],\n                            )\n                            description[\"oml:run\"][\"oml:output_data\"][\"oml:evaluation\"].append(\n                                current,\n                            )\n        return description\n</code></pre>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>The ID of the run, None if not uploaded to the server yet.</p>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.predictions","title":"<code>predictions: pd.DataFrame</code>  <code>property</code>","text":"<p>Return a DataFrame with predictions for this run</p>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.from_filesystem","title":"<code>from_filesystem(directory, expect_model=True)</code>  <code>classmethod</code>","text":"<p>The inverse of the to_filesystem method. Instantiates an OpenMLRun object based on files stored on the file system.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results are stored</p> required <code>expect_model</code> <code>bool</code> <p>if True, it requires the model pickle to be present, and an error will be thrown if not. Otherwise, the model might or might not be present.</p> <code>True</code> <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRun</code> <p>the re-instantiated run object</p> Source code in <code>openml/runs/run.py</code> <pre><code>@classmethod\ndef from_filesystem(cls, directory: str | Path, expect_model: bool = True) -&gt; OpenMLRun:  # noqa: FBT001, FBT002\n    \"\"\"\n    The inverse of the to_filesystem method. Instantiates an OpenMLRun\n    object based on files stored on the file system.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        are stored\n\n    expect_model : bool\n        if True, it requires the model pickle to be present, and an error\n        will be thrown if not. Otherwise, the model might or might not\n        be present.\n\n    Returns\n    -------\n    run : OpenMLRun\n        the re-instantiated run object\n    \"\"\"\n    # Avoiding cyclic imports\n    import openml.runs.functions\n\n    directory = Path(directory)\n    if not directory.is_dir():\n        raise ValueError(\"Could not find folder\")\n\n    description_path = directory / \"description.xml\"\n    predictions_path = directory / \"predictions.arff\"\n    trace_path = directory / \"trace.arff\"\n    model_path = directory / \"model.pkl\"\n\n    if not description_path.is_file():\n        raise ValueError(\"Could not find description.xml\")\n    if not predictions_path.is_file():\n        raise ValueError(\"Could not find predictions.arff\")\n    if (not model_path.is_file()) and expect_model:\n        raise ValueError(\"Could not find model.pkl\")\n\n    with description_path.open() as fht:\n        xml_string = fht.read()\n    run = openml.runs.functions._create_run_from_xml(xml_string, from_server=False)\n\n    if run.flow_id is None:\n        flow = openml.flows.OpenMLFlow.from_filesystem(directory)\n        run.flow = flow\n        run.flow_name = flow.name\n\n    with predictions_path.open() as fht:\n        predictions = arff.load(fht)\n        run.data_content = predictions[\"data\"]\n\n    if model_path.is_file():\n        # note that it will load the model if the file exists, even if\n        # expect_model is False\n        with model_path.open(\"rb\") as fhb:\n            run.model = pickle.load(fhb)  # noqa: S301\n\n    if trace_path.is_file():\n        run.trace = openml.runs.OpenMLRunTrace._from_filesystem(trace_path)\n\n    return run\n</code></pre>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.get_metric_fn","title":"<code>get_metric_fn(sklearn_fn, kwargs=None)</code>","text":"<p>Calculates metric scores based on predicted values. Assumes the run has been executed locally (and contains run_data). Furthermore, it assumes that the 'correct' or 'truth' attribute is specified in the arff (which is an optional field, but always the case for openml-python runs)</p> <p>Parameters:</p> Name Type Description Default <code>sklearn_fn</code> <code>function</code> <p>a function pointer to a sklearn function that accepts <code>y_true</code>, <code>y_pred</code> and <code>**kwargs</code></p> required <code>kwargs</code> <code>dict</code> <p>kwargs for the function</p> <code>None</code> <p>Returns:</p> Name Type Description <code>scores</code> <code>ndarray of scores of length num_folds * num_repeats</code> <p>metric results</p> Source code in <code>openml/runs/run.py</code> <pre><code>def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -&gt; np.ndarray:  # noqa: PLR0915, PLR0912, C901\n    \"\"\"Calculates metric scores based on predicted values. Assumes the\n    run has been executed locally (and contains run_data). Furthermore,\n    it assumes that the 'correct' or 'truth' attribute is specified in\n    the arff (which is an optional field, but always the case for\n    openml-python runs)\n\n    Parameters\n    ----------\n    sklearn_fn : function\n        a function pointer to a sklearn function that\n        accepts ``y_true``, ``y_pred`` and ``**kwargs``\n    kwargs : dict\n        kwargs for the function\n\n    Returns\n    -------\n    scores : ndarray of scores of length num_folds * num_repeats\n        metric results\n    \"\"\"\n    kwargs = kwargs if kwargs else {}\n    if self.data_content is not None and self.task_id is not None:\n        predictions_arff = self._generate_arff_dict()\n    elif (self.output_files is not None) and (\"predictions\" in self.output_files):\n        predictions_file_url = openml._api_calls._file_id_to_url(\n            self.output_files[\"predictions\"],\n            \"predictions.arff\",\n        )\n        response = openml._api_calls._download_text_file(predictions_file_url)\n        predictions_arff = arff.loads(response)\n        # TODO: make this a stream reader\n    else:\n        raise ValueError(\n            \"Run should have been locally executed or \" \"contain outputfile reference.\",\n        )\n\n    # Need to know more about the task to compute scores correctly\n    task = get_task(self.task_id)\n\n    attribute_names = [att[0] for att in predictions_arff[\"attributes\"]]\n    if (\n        task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE]\n        and \"correct\" not in attribute_names\n    ):\n        raise ValueError('Attribute \"correct\" should be set for ' \"classification task runs\")\n    if task.task_type_id == TaskType.SUPERVISED_REGRESSION and \"truth\" not in attribute_names:\n        raise ValueError('Attribute \"truth\" should be set for ' \"regression task runs\")\n    if task.task_type_id != TaskType.CLUSTERING and \"prediction\" not in attribute_names:\n        raise ValueError('Attribute \"predict\" should be set for ' \"supervised task runs\")\n\n    def _attribute_list_to_dict(attribute_list):  # type: ignore\n        # convenience function: Creates a mapping to map from the name of\n        # attributes present in the arff prediction file to their index.\n        # This is necessary because the number of classes can be different\n        # for different tasks.\n        res = OrderedDict()\n        for idx in range(len(attribute_list)):\n            res[attribute_list[idx][0]] = idx\n        return res\n\n    attribute_dict = _attribute_list_to_dict(predictions_arff[\"attributes\"])\n\n    repeat_idx = attribute_dict[\"repeat\"]\n    fold_idx = attribute_dict[\"fold\"]\n    predicted_idx = attribute_dict[\"prediction\"]  # Assume supervised task\n\n    if task.task_type_id in (TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE):\n        correct_idx = attribute_dict[\"correct\"]\n    elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n        correct_idx = attribute_dict[\"truth\"]\n    has_samples = False\n    if \"sample\" in attribute_dict:\n        sample_idx = attribute_dict[\"sample\"]\n        has_samples = True\n\n    if (\n        predictions_arff[\"attributes\"][predicted_idx][1]\n        != predictions_arff[\"attributes\"][correct_idx][1]\n    ):\n        pred = predictions_arff[\"attributes\"][predicted_idx][1]\n        corr = predictions_arff[\"attributes\"][correct_idx][1]\n        raise ValueError(\n            \"Predicted and Correct do not have equal values:\" f\" {pred!s} Vs. {corr!s}\",\n        )\n\n    # TODO: these could be cached\n    values_predict: dict[int, dict[int, dict[int, list[float]]]] = {}\n    values_correct: dict[int, dict[int, dict[int, list[float]]]] = {}\n    for _line_idx, line in enumerate(predictions_arff[\"data\"]):\n        rep = line[repeat_idx]\n        fold = line[fold_idx]\n        samp = line[sample_idx] if has_samples else 0\n\n        if task.task_type_id in [\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.LEARNING_CURVE,\n        ]:\n            prediction = predictions_arff[\"attributes\"][predicted_idx][1].index(\n                line[predicted_idx],\n            )\n            correct = predictions_arff[\"attributes\"][predicted_idx][1].index(line[correct_idx])\n        elif task.task_type_id == TaskType.SUPERVISED_REGRESSION:\n            prediction = line[predicted_idx]\n            correct = line[correct_idx]\n        if rep not in values_predict:\n            values_predict[rep] = OrderedDict()\n            values_correct[rep] = OrderedDict()\n        if fold not in values_predict[rep]:\n            values_predict[rep][fold] = OrderedDict()\n            values_correct[rep][fold] = OrderedDict()\n        if samp not in values_predict[rep][fold]:\n            values_predict[rep][fold][samp] = []\n            values_correct[rep][fold][samp] = []\n\n        values_predict[rep][fold][samp].append(prediction)\n        values_correct[rep][fold][samp].append(correct)\n\n    scores = []\n    for rep in values_predict:\n        for fold in values_predict[rep]:\n            last_sample = len(values_predict[rep][fold]) - 1\n            y_pred = values_predict[rep][fold][last_sample]\n            y_true = values_correct[rep][fold][last_sample]\n            scores.append(sklearn_fn(y_true, y_pred, **kwargs))\n    return np.array(scores)\n</code></pre>"},{"location":"reference/runs/run/#openml.runs.run.OpenMLRun.to_filesystem","title":"<code>to_filesystem(directory, store_model=True)</code>","text":"<p>The inverse of the from_filesystem method. Serializes a run on the filesystem, to be uploaded later.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>str</code> <p>a path leading to the folder where the results will be stored. Should be empty</p> required <code>store_model</code> <code>(bool, optional(default=True))</code> <p>if True, a model will be pickled as well. As this is the most storage expensive part, it is often desirable to not store the model.</p> <code>True</code> Source code in <code>openml/runs/run.py</code> <pre><code>def to_filesystem(\n    self,\n    directory: str | Path,\n    store_model: bool = True,  # noqa: FBT001, FBT002\n) -&gt; None:\n    \"\"\"\n    The inverse of the from_filesystem method. Serializes a run\n    on the filesystem, to be uploaded later.\n\n    Parameters\n    ----------\n    directory : str\n        a path leading to the folder where the results\n        will be stored. Should be empty\n\n    store_model : bool, optional (default=True)\n        if True, a model will be pickled as well. As this is the most\n        storage expensive part, it is often desirable to not store the\n        model.\n    \"\"\"\n    if self.data_content is None or self.model is None:\n        raise ValueError(\"Run should have been executed (and contain \" \"model / predictions)\")\n    directory = Path(directory)\n    directory.mkdir(exist_ok=True, parents=True)\n\n    if any(directory.iterdir()):\n        raise ValueError(f\"Output directory {directory.expanduser().resolve()} should be empty\")\n\n    run_xml = self._to_xml()\n    predictions_arff = arff.dumps(self._generate_arff_dict())\n\n    # It seems like typing does not allow to define the same variable multiple times\n    with (directory / \"description.xml\").open(\"w\") as fh:\n        fh.write(run_xml)\n    with (directory / \"predictions.arff\").open(\"w\") as fh:\n        fh.write(predictions_arff)\n    if store_model:\n        with (directory / \"model.pkl\").open(\"wb\") as fh_b:\n            pickle.dump(self.model, fh_b)\n\n    if self.flow_id is None and self.flow is not None:\n        self.flow.to_filesystem(directory)\n\n    if self.trace is not None:\n        self.trace._to_filesystem(directory)\n</code></pre>"},{"location":"reference/runs/trace/","title":"trace","text":""},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace","title":"<code>OpenMLRunTrace</code>","text":"<p>OpenML Run Trace: parsed output from Run Trace call</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>OpenML run id.</p> required <code>trace_iterations</code> <code>dict</code> <p>Mapping from key <code>(repeat, fold, iteration)</code> to an object of OpenMLTraceIteration.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>class OpenMLRunTrace:\n    \"\"\"OpenML Run Trace: parsed output from Run Trace call\n\n    Parameters\n    ----------\n    run_id : int\n        OpenML run id.\n\n    trace_iterations : dict\n        Mapping from key ``(repeat, fold, iteration)`` to an object of\n        OpenMLTraceIteration.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        run_id: int | None,\n        trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n    ):\n        \"\"\"Object to hold the trace content of a run.\n\n        Parameters\n        ----------\n        run_id : int\n            Id for which the trace content is to be stored.\n        trace_iterations : List[List]\n            The trace content obtained by running a flow on a task.\n        \"\"\"\n        self.run_id = run_id\n        self.trace_iterations = trace_iterations\n\n    def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n        \"\"\"\n        Returns the trace iteration that was marked as selected. In\n        case multiple are marked as selected (should not happen) the\n        first of these is returned\n\n        Parameters\n        ----------\n        fold: int\n\n        repeat: int\n\n        Returns\n        -------\n        int\n            The trace iteration from the given fold and repeat that was\n            selected as the best iteration by the search procedure\n        \"\"\"\n        for r, f, i in self.trace_iterations:\n            if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n                return i\n        raise ValueError(\n            \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n        )\n\n    @classmethod\n    def generate(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n    ) -&gt; OpenMLRunTrace:\n        \"\"\"Generates an OpenMLRunTrace.\n\n        Generates the trace object from the attributes and content extracted\n        while running the underlying flow.\n\n        Parameters\n        ----------\n        attributes : list\n            List of tuples describing the arff attributes.\n\n        content : list\n            List of lists containing information about the individual tuning\n            runs.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        if content is None:\n            raise ValueError(\"Trace content not available.\")\n        if attributes is None:\n            raise ValueError(\"Trace attributes not available.\")\n        if len(content) == 0:\n            raise ValueError(\"Trace content is empty.\")\n        if len(attributes) != len(content[0]):\n            raise ValueError(\n                \"Trace_attributes and trace_content not compatible:\"\n                f\" {attributes} vs {content[0]}\",\n            )\n\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not allowed when constructing a \"\n            \"trace object from run results.\",\n        )\n\n    @classmethod\n    def _from_filesystem(cls, file_path: str | Path) -&gt; OpenMLRunTrace:\n        \"\"\"\n        Logic to deserialize the trace from the filesystem.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff is stored.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        file_path = Path(file_path)\n\n        if not file_path.exists():\n            raise ValueError(\"Trace file doesn't exist\")\n\n        with file_path.open(\"r\") as fp:\n            trace_arff = arff.load(fp)\n\n        for trace_idx in range(len(trace_arff[\"data\"])):\n            # iterate over first three entrees of a trace row\n            # (fold, repeat, trace_iteration) these should be int\n            for line_idx in range(3):\n                trace_arff[\"data\"][trace_idx][line_idx] = int(\n                    trace_arff[\"data\"][trace_idx][line_idx],\n                )\n\n        return cls.trace_from_arff(trace_arff)\n\n    def _to_filesystem(self, file_path: str | Path) -&gt; None:\n        \"\"\"Serialize the trace object to the filesystem.\n\n        Serialize the trace object as an arff.\n\n        Parameters\n        ----------\n        file_path: str | Path\n            File path where the trace arff will be stored.\n        \"\"\"\n        trace_path = Path(file_path) / \"trace.arff\"\n\n        trace_arff = arff.dumps(self.trace_to_arff())\n        with trace_path.open(\"w\") as f:\n            f.write(trace_arff)\n\n    def trace_to_arff(self) -&gt; dict[str, Any]:\n        \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n        Uses the trace object to generate an arff dictionary representation.\n\n        Returns\n        -------\n        arff_dict : dict\n            Dictionary representation of the ARFF file that will be uploaded.\n            Contains information about the optimization trace.\n        \"\"\"\n        if self.trace_iterations is None:\n            raise ValueError(\"trace_iterations missing from the trace object\")\n\n        # attributes that will be in trace arff\n        trace_attributes = [\n            (\"repeat\", \"NUMERIC\"),\n            (\"fold\", \"NUMERIC\"),\n            (\"iteration\", \"NUMERIC\"),\n            (\"evaluation\", \"NUMERIC\"),\n            (\"selected\", [\"true\", \"false\"]),\n        ]\n        trace_attributes.extend(\n            [\n                (PREFIX + parameter, \"STRING\")\n                for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n            ],\n        )\n\n        arff_dict: dict[str, Any] = {}\n        data = []\n        for trace_iteration in self.trace_iterations.values():\n            tmp_list = []\n            for _attr, _ in trace_attributes:\n                if _attr.startswith(PREFIX):\n                    attr = _attr[len(PREFIX) :]\n                    value = trace_iteration.get_parameters()[attr]\n                else:\n                    attr = _attr\n                    value = getattr(trace_iteration, attr)\n\n                if attr == \"selected\":\n                    tmp_list.append(\"true\" if value else \"false\")\n                else:\n                    tmp_list.append(value)\n            data.append(tmp_list)\n\n        arff_dict[\"attributes\"] = trace_attributes\n        arff_dict[\"data\"] = data\n        # TODO allow to pass a trace description when running a flow\n        arff_dict[\"relation\"] = \"Trace\"\n        return arff_dict\n\n    @classmethod\n    def trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from arff trace.\n\n        Creates a trace file from arff object (for example, generated by a\n        local run).\n\n        Parameters\n        ----------\n        arff_obj : dict\n            LIAC arff obj, dict containing attributes, relation, data.\n\n        Returns\n        -------\n        OpenMLRunTrace\n        \"\"\"\n        attributes = arff_obj[\"attributes\"]\n        content = arff_obj[\"data\"]\n        return cls._trace_from_arff_struct(\n            attributes=attributes,\n            content=content,\n            error_message=\"setup_string not supported for arff serialization\",\n        )\n\n    @classmethod\n    def _trace_from_arff_struct(\n        cls,\n        attributes: list[tuple[str, str]],\n        content: list[list[int | float | str]],\n        error_message: str,\n    ) -&gt; Self:\n        \"\"\"Generate a trace dictionary from ARFF structure.\n\n        Parameters\n        ----------\n        cls : type\n            The trace object to be created.\n        attributes : list[tuple[str, str]]\n            Attribute descriptions.\n        content : list[list[int | float | str]]]\n            List of instances.\n        error_message : str\n            Error message to raise if `setup_string` is in `attributes`.\n\n        Returns\n        -------\n        OrderedDict\n            A dictionary representing the trace.\n        \"\"\"\n        trace = OrderedDict()\n        attribute_idx = {att[0]: idx for idx, att in enumerate(attributes)}\n\n        for required_attribute in REQUIRED_ATTRIBUTES:\n            if required_attribute not in attribute_idx:\n                raise ValueError(\"arff misses required attribute: %s\" % required_attribute)\n        if \"setup_string\" in attribute_idx:\n            raise ValueError(error_message)\n\n        # note that the required attributes can not be duplicated because\n        # they are not parameters\n        parameter_attributes = []\n        for attribute in attribute_idx:\n            if attribute in REQUIRED_ATTRIBUTES or attribute == \"setup_string\":\n                continue\n\n            if not attribute.startswith(PREFIX):\n                raise ValueError(\n                    f\"Encountered unknown attribute {attribute} that does not start \"\n                    f\"with prefix {PREFIX}\",\n                )\n\n            parameter_attributes.append(attribute)\n\n        for itt in content:\n            repeat = int(itt[attribute_idx[\"repeat\"]])\n            fold = int(itt[attribute_idx[\"fold\"]])\n            iteration = int(itt[attribute_idx[\"iteration\"]])\n            evaluation = float(itt[attribute_idx[\"evaluation\"]])\n            selected_value = itt[attribute_idx[\"selected\"]]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for selected field, '\n                    \"received: %s\" % selected_value,\n                )\n\n            parameters = {\n                attribute: itt[attribute_idx[attribute]] for attribute in parameter_attributes\n            }\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=None,\n                evaluation=evaluation,\n                selected=selected,\n                parameters=parameters,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(None, trace)\n\n    @classmethod\n    def trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n        \"\"\"Generate trace from xml.\n\n        Creates a trace file from the xml description.\n\n        Parameters\n        ----------\n        xml : string | file-like object\n            An xml description that can be either a `string` or a file-like\n            object.\n\n        Returns\n        -------\n        run : OpenMLRunTrace\n            Object containing the run id and a dict containing the trace\n            iterations.\n        \"\"\"\n        if isinstance(xml, Path):\n            xml = str(xml.absolute())\n\n        result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n        run_id = result_dict[\"oml:run_id\"]\n        trace = OrderedDict()\n\n        if \"oml:trace_iteration\" not in result_dict:\n            raise ValueError(\"Run does not contain valid trace. \")\n        if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n            raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n        for itt in result_dict[\"oml:trace_iteration\"]:\n            repeat = int(itt[\"oml:repeat\"])\n            fold = int(itt[\"oml:fold\"])\n            iteration = int(itt[\"oml:iteration\"])\n            setup_string = json.loads(itt[\"oml:setup_string\"])\n            evaluation = float(itt[\"oml:evaluation\"])\n            selected_value = itt[\"oml:selected\"]\n            if selected_value == \"true\":\n                selected = True\n            elif selected_value == \"false\":\n                selected = False\n            else:\n                raise ValueError(\n                    'expected {\"true\", \"false\"} value for '\n                    \"selected field, received: %s\" % selected_value,\n                )\n\n            current = OpenMLTraceIteration(\n                repeat=repeat,\n                fold=fold,\n                iteration=iteration,\n                setup_string=setup_string,\n                evaluation=evaluation,\n                selected=selected,\n            )\n            trace[(repeat, fold, iteration)] = current\n\n        return cls(run_id, trace)\n\n    @classmethod\n    def merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n        \"\"\"Merge multiple traces into a single trace.\n\n        Parameters\n        ----------\n        cls : type\n            Type of the trace object to be created.\n        traces : List[OpenMLRunTrace]\n            List of traces to merge.\n\n        Returns\n        -------\n        OpenMLRunTrace\n            A trace object representing the merged traces.\n\n        Raises\n        ------\n        ValueError\n            If the parameters in the iterations of the traces being merged are not equal.\n            If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n        \"\"\"\n        merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n        previous_iteration = None\n        for trace in traces:\n            for iteration in trace:\n                key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n                assert iteration.parameters is not None\n                param_keys = iteration.parameters.keys()\n\n                if previous_iteration is not None:\n                    trace_itr = merged_trace[previous_iteration]\n\n                    assert trace_itr.parameters is not None\n                    trace_itr_keys = trace_itr.parameters.keys()\n\n                    if list(param_keys) != list(trace_itr_keys):\n                        raise ValueError(\n                            \"Cannot merge traces because the parameters are not equal: \"\n                            \"{} vs {}\".format(\n                                list(trace_itr.parameters.keys()),\n                                list(iteration.parameters.keys()),\n                            ),\n                        )\n\n                if key in merged_trace:\n                    raise ValueError(\n                        f\"Cannot merge traces because key '{key}' was encountered twice\",\n                    )\n\n                merged_trace[key] = iteration\n                previous_iteration = key\n\n        return cls(None, merged_trace)\n\n    def __repr__(self) -&gt; str:\n        return \"[Run id: {}, {} trace iterations]\".format(\n            -1 if self.run_id is None else self.run_id,\n            len(self.trace_iterations),\n        )\n\n    def __iter__(self) -&gt; Iterator[OpenMLTraceIteration]:\n        yield from self.trace_iterations.values()\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.__init__","title":"<code>__init__(run_id, trace_iterations)</code>","text":"<p>Object to hold the trace content of a run.</p> <p>Parameters:</p> Name Type Description Default <code>run_id</code> <code>int</code> <p>Id for which the trace content is to be stored.</p> required <code>trace_iterations</code> <code>List[List]</code> <p>The trace content obtained by running a flow on a task.</p> required Source code in <code>openml/runs/trace.py</code> <pre><code>def __init__(\n    self,\n    run_id: int | None,\n    trace_iterations: dict[tuple[int, int, int], OpenMLTraceIteration],\n):\n    \"\"\"Object to hold the trace content of a run.\n\n    Parameters\n    ----------\n    run_id : int\n        Id for which the trace content is to be stored.\n    trace_iterations : List[List]\n        The trace content obtained by running a flow on a task.\n    \"\"\"\n    self.run_id = run_id\n    self.trace_iterations = trace_iterations\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.generate","title":"<code>generate(attributes, content)</code>  <code>classmethod</code>","text":"<p>Generates an OpenMLRunTrace.</p> <p>Generates the trace object from the attributes and content extracted while running the underlying flow.</p> <p>Parameters:</p> Name Type Description Default <code>attributes</code> <code>list</code> <p>List of tuples describing the arff attributes.</p> required <code>content</code> <code>list</code> <p>List of lists containing information about the individual tuning runs.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef generate(\n    cls,\n    attributes: list[tuple[str, str]],\n    content: list[list[int | float | str]],\n) -&gt; OpenMLRunTrace:\n    \"\"\"Generates an OpenMLRunTrace.\n\n    Generates the trace object from the attributes and content extracted\n    while running the underlying flow.\n\n    Parameters\n    ----------\n    attributes : list\n        List of tuples describing the arff attributes.\n\n    content : list\n        List of lists containing information about the individual tuning\n        runs.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    if content is None:\n        raise ValueError(\"Trace content not available.\")\n    if attributes is None:\n        raise ValueError(\"Trace attributes not available.\")\n    if len(content) == 0:\n        raise ValueError(\"Trace content is empty.\")\n    if len(attributes) != len(content[0]):\n        raise ValueError(\n            \"Trace_attributes and trace_content not compatible:\"\n            f\" {attributes} vs {content[0]}\",\n        )\n\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not allowed when constructing a \"\n        \"trace object from run results.\",\n    )\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.get_selected_iteration","title":"<code>get_selected_iteration(fold, repeat)</code>","text":"<p>Returns the trace iteration that was marked as selected. In case multiple are marked as selected (should not happen) the first of these is returned</p> <p>Parameters:</p> Name Type Description Default <code>fold</code> <code>int</code> required <code>repeat</code> <code>int</code> required <p>Returns:</p> Type Description <code>int</code> <p>The trace iteration from the given fold and repeat that was selected as the best iteration by the search procedure</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_selected_iteration(self, fold: int, repeat: int) -&gt; int:\n    \"\"\"\n    Returns the trace iteration that was marked as selected. In\n    case multiple are marked as selected (should not happen) the\n    first of these is returned\n\n    Parameters\n    ----------\n    fold: int\n\n    repeat: int\n\n    Returns\n    -------\n    int\n        The trace iteration from the given fold and repeat that was\n        selected as the best iteration by the search procedure\n    \"\"\"\n    for r, f, i in self.trace_iterations:\n        if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True:\n            return i\n    raise ValueError(\n        \"Could not find the selected iteration for rep/fold %d/%d\" % (repeat, fold),\n    )\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.merge_traces","title":"<code>merge_traces(traces)</code>  <code>classmethod</code>","text":"<p>Merge multiple traces into a single trace.</p> <p>Parameters:</p> Name Type Description Default <code>cls</code> <code>type</code> <p>Type of the trace object to be created.</p> required <code>traces</code> <code>List[OpenMLRunTrace]</code> <p>List of traces to merge.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> <p>A trace object representing the merged traces.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the parameters in the iterations of the traces being merged are not equal. If a key (repeat, fold, iteration) is encountered twice while merging the traces.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef merge_traces(cls, traces: list[OpenMLRunTrace]) -&gt; OpenMLRunTrace:\n    \"\"\"Merge multiple traces into a single trace.\n\n    Parameters\n    ----------\n    cls : type\n        Type of the trace object to be created.\n    traces : List[OpenMLRunTrace]\n        List of traces to merge.\n\n    Returns\n    -------\n    OpenMLRunTrace\n        A trace object representing the merged traces.\n\n    Raises\n    ------\n    ValueError\n        If the parameters in the iterations of the traces being merged are not equal.\n        If a key (repeat, fold, iteration) is encountered twice while merging the traces.\n    \"\"\"\n    merged_trace: dict[tuple[int, int, int], OpenMLTraceIteration] = {}\n\n    previous_iteration = None\n    for trace in traces:\n        for iteration in trace:\n            key = (iteration.repeat, iteration.fold, iteration.iteration)\n\n            assert iteration.parameters is not None\n            param_keys = iteration.parameters.keys()\n\n            if previous_iteration is not None:\n                trace_itr = merged_trace[previous_iteration]\n\n                assert trace_itr.parameters is not None\n                trace_itr_keys = trace_itr.parameters.keys()\n\n                if list(param_keys) != list(trace_itr_keys):\n                    raise ValueError(\n                        \"Cannot merge traces because the parameters are not equal: \"\n                        \"{} vs {}\".format(\n                            list(trace_itr.parameters.keys()),\n                            list(iteration.parameters.keys()),\n                        ),\n                    )\n\n            if key in merged_trace:\n                raise ValueError(\n                    f\"Cannot merge traces because key '{key}' was encountered twice\",\n                )\n\n            merged_trace[key] = iteration\n            previous_iteration = key\n\n    return cls(None, merged_trace)\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.trace_from_arff","title":"<code>trace_from_arff(arff_obj)</code>  <code>classmethod</code>","text":"<p>Generate trace from arff trace.</p> <p>Creates a trace file from arff object (for example, generated by a local run).</p> <p>Parameters:</p> Name Type Description Default <code>arff_obj</code> <code>dict</code> <p>LIAC arff obj, dict containing attributes, relation, data.</p> required <p>Returns:</p> Type Description <code>OpenMLRunTrace</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_arff(cls, arff_obj: dict[str, Any]) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from arff trace.\n\n    Creates a trace file from arff object (for example, generated by a\n    local run).\n\n    Parameters\n    ----------\n    arff_obj : dict\n        LIAC arff obj, dict containing attributes, relation, data.\n\n    Returns\n    -------\n    OpenMLRunTrace\n    \"\"\"\n    attributes = arff_obj[\"attributes\"]\n    content = arff_obj[\"data\"]\n    return cls._trace_from_arff_struct(\n        attributes=attributes,\n        content=content,\n        error_message=\"setup_string not supported for arff serialization\",\n    )\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.trace_from_xml","title":"<code>trace_from_xml(xml)</code>  <code>classmethod</code>","text":"<p>Generate trace from xml.</p> <p>Creates a trace file from the xml description.</p> <p>Parameters:</p> Name Type Description Default <code>xml</code> <code>string | file-like object</code> <p>An xml description that can be either a <code>string</code> or a file-like object.</p> required <p>Returns:</p> Name Type Description <code>run</code> <code>OpenMLRunTrace</code> <p>Object containing the run id and a dict containing the trace iterations.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>@classmethod\ndef trace_from_xml(cls, xml: str | Path | IO) -&gt; OpenMLRunTrace:\n    \"\"\"Generate trace from xml.\n\n    Creates a trace file from the xml description.\n\n    Parameters\n    ----------\n    xml : string | file-like object\n        An xml description that can be either a `string` or a file-like\n        object.\n\n    Returns\n    -------\n    run : OpenMLRunTrace\n        Object containing the run id and a dict containing the trace\n        iterations.\n    \"\"\"\n    if isinstance(xml, Path):\n        xml = str(xml.absolute())\n\n    result_dict = xmltodict.parse(xml, force_list=(\"oml:trace_iteration\",))[\"oml:trace\"]\n\n    run_id = result_dict[\"oml:run_id\"]\n    trace = OrderedDict()\n\n    if \"oml:trace_iteration\" not in result_dict:\n        raise ValueError(\"Run does not contain valid trace. \")\n    if not isinstance(result_dict[\"oml:trace_iteration\"], list):\n        raise TypeError(type(result_dict[\"oml:trace_iteration\"]))\n\n    for itt in result_dict[\"oml:trace_iteration\"]:\n        repeat = int(itt[\"oml:repeat\"])\n        fold = int(itt[\"oml:fold\"])\n        iteration = int(itt[\"oml:iteration\"])\n        setup_string = json.loads(itt[\"oml:setup_string\"])\n        evaluation = float(itt[\"oml:evaluation\"])\n        selected_value = itt[\"oml:selected\"]\n        if selected_value == \"true\":\n            selected = True\n        elif selected_value == \"false\":\n            selected = False\n        else:\n            raise ValueError(\n                'expected {\"true\", \"false\"} value for '\n                \"selected field, received: %s\" % selected_value,\n            )\n\n        current = OpenMLTraceIteration(\n            repeat=repeat,\n            fold=fold,\n            iteration=iteration,\n            setup_string=setup_string,\n            evaluation=evaluation,\n            selected=selected,\n        )\n        trace[(repeat, fold, iteration)] = current\n\n    return cls(run_id, trace)\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLRunTrace.trace_to_arff","title":"<code>trace_to_arff()</code>","text":"<p>Generate the arff dictionary for uploading predictions to the server.</p> <p>Uses the trace object to generate an arff dictionary representation.</p> <p>Returns:</p> Name Type Description <code>arff_dict</code> <code>dict</code> <p>Dictionary representation of the ARFF file that will be uploaded. Contains information about the optimization trace.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def trace_to_arff(self) -&gt; dict[str, Any]:\n    \"\"\"Generate the arff dictionary for uploading predictions to the server.\n\n    Uses the trace object to generate an arff dictionary representation.\n\n    Returns\n    -------\n    arff_dict : dict\n        Dictionary representation of the ARFF file that will be uploaded.\n        Contains information about the optimization trace.\n    \"\"\"\n    if self.trace_iterations is None:\n        raise ValueError(\"trace_iterations missing from the trace object\")\n\n    # attributes that will be in trace arff\n    trace_attributes = [\n        (\"repeat\", \"NUMERIC\"),\n        (\"fold\", \"NUMERIC\"),\n        (\"iteration\", \"NUMERIC\"),\n        (\"evaluation\", \"NUMERIC\"),\n        (\"selected\", [\"true\", \"false\"]),\n    ]\n    trace_attributes.extend(\n        [\n            (PREFIX + parameter, \"STRING\")\n            for parameter in next(iter(self.trace_iterations.values())).get_parameters()\n        ],\n    )\n\n    arff_dict: dict[str, Any] = {}\n    data = []\n    for trace_iteration in self.trace_iterations.values():\n        tmp_list = []\n        for _attr, _ in trace_attributes:\n            if _attr.startswith(PREFIX):\n                attr = _attr[len(PREFIX) :]\n                value = trace_iteration.get_parameters()[attr]\n            else:\n                attr = _attr\n                value = getattr(trace_iteration, attr)\n\n            if attr == \"selected\":\n                tmp_list.append(\"true\" if value else \"false\")\n            else:\n                tmp_list.append(value)\n        data.append(tmp_list)\n\n    arff_dict[\"attributes\"] = trace_attributes\n    arff_dict[\"data\"] = data\n    # TODO allow to pass a trace description when running a flow\n    arff_dict[\"relation\"] = \"Trace\"\n    return arff_dict\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLTraceIteration","title":"<code>OpenMLTraceIteration</code>  <code>dataclass</code>","text":"<p>OpenML Trace Iteration: parsed output from Run Trace call Exactly one of <code>setup_string</code> or <code>parameters</code> must be provided.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>repeat number (in case of no repeats: 0)</p> required <code>fold</code> <code>int</code> <p>fold number (in case of no folds: 0)</p> required <code>iteration</code> <code>int</code> <p>iteration number of optimization procedure</p> required <code>setup_string</code> <code>str</code> <p>json string representing the parameters If not provided, <code>parameters</code> should be set.</p> <code>None</code> <code>evaluation</code> <code>double</code> <p>The evaluation that was awarded to this trace iteration. Measure is defined by the task</p> required <code>selected</code> <code>bool</code> <p>Whether this was the best of all iterations, and hence selected for making predictions. Per fold/repeat there should be only one iteration selected</p> required <code>parameters</code> <code>OrderedDict</code> <p>Dictionary specifying parameter names and their values. If not provided, <code>setup_string</code> should be set.</p> <code>None</code> Source code in <code>openml/runs/trace.py</code> <pre><code>@dataclass\nclass OpenMLTraceIteration:\n    \"\"\"\n    OpenML Trace Iteration: parsed output from Run Trace call\n    Exactly one of `setup_string` or `parameters` must be provided.\n\n    Parameters\n    ----------\n    repeat : int\n        repeat number (in case of no repeats: 0)\n\n    fold : int\n        fold number (in case of no folds: 0)\n\n    iteration : int\n        iteration number of optimization procedure\n\n    setup_string : str, optional\n        json string representing the parameters\n        If not provided, ``parameters`` should be set.\n\n    evaluation : double\n        The evaluation that was awarded to this trace iteration.\n        Measure is defined by the task\n\n    selected : bool\n        Whether this was the best of all iterations, and hence\n        selected for making predictions. Per fold/repeat there\n        should be only one iteration selected\n\n    parameters : OrderedDict, optional\n        Dictionary specifying parameter names and their values.\n        If not provided, ``setup_string`` should be set.\n    \"\"\"\n\n    repeat: int\n    fold: int\n    iteration: int\n\n    evaluation: float\n    selected: bool\n\n    setup_string: dict[str, str] | None = None\n    parameters: dict[str, str | int | float] | None = None\n\n    def __post_init__(self) -&gt; None:\n        # TODO: refactor into one argument of type &lt;str | OrderedDict&gt;\n        if self.setup_string and self.parameters:\n            raise ValueError(\n                \"Can only be instantiated with either `setup_string` or `parameters` argument.\",\n            )\n\n        if not (self.setup_string or self.parameters):\n            raise ValueError(\n                \"Either `setup_string` or `parameters` needs to be passed as argument.\",\n            )\n\n        if self.parameters is not None and not isinstance(self.parameters, dict):\n            raise TypeError(\n                \"argument parameters is not an instance of OrderedDict, but %s\"\n                % str(type(self.parameters)),\n            )\n\n    def get_parameters(self) -&gt; dict[str, Any]:\n        \"\"\"Get the parameters of this trace iteration.\"\"\"\n        # parameters have prefix 'parameter_'\n        if self.setup_string:\n            return {\n                param[len(PREFIX) :]: json.loads(value)\n                for param, value in self.setup_string.items()\n            }\n\n        assert self.parameters is not None\n        return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/runs/trace/#openml.runs.trace.OpenMLTraceIteration.get_parameters","title":"<code>get_parameters()</code>","text":"<p>Get the parameters of this trace iteration.</p> Source code in <code>openml/runs/trace.py</code> <pre><code>def get_parameters(self) -&gt; dict[str, Any]:\n    \"\"\"Get the parameters of this trace iteration.\"\"\"\n    # parameters have prefix 'parameter_'\n    if self.setup_string:\n        return {\n            param[len(PREFIX) :]: json.loads(value)\n            for param, value in self.setup_string.items()\n        }\n\n    assert self.parameters is not None\n    return {param[len(PREFIX) :]: value for param, value in self.parameters.items()}\n</code></pre>"},{"location":"reference/setups/","title":"setups","text":""},{"location":"reference/setups/#openml.setups.OpenMLParameter","title":"<code>OpenMLParameter</code>","text":"<p>Parameter object (used in setup).</p> <p>Parameters:</p> Name Type Description Default <code>input_id</code> <code>int</code> <p>The input id from the openml database</p> required <code>flow</code> <p>The flow to which this parameter is associated</p> required <code>flow</code> <p>The name of the flow (no version number) to which this parameter is associated</p> required <code>full_name</code> <code>str</code> <p>The name of the flow and parameter combined</p> required <code>parameter_name</code> <code>str</code> <p>The name of the parameter</p> required <code>data_type</code> <code>str</code> <p>The datatype of the parameter. generally unused for sklearn flows</p> required <code>default_value</code> <code>str</code> <p>The default value. For sklearn parameters, this is unknown and a default value is selected arbitrarily</p> required <code>value</code> <code>str</code> <p>If the parameter was set, the value that it was set to.</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLParameter:\n    \"\"\"Parameter object (used in setup).\n\n    Parameters\n    ----------\n    input_id : int\n        The input id from the openml database\n    flow id : int\n        The flow to which this parameter is associated\n    flow name : str\n        The name of the flow (no version number) to which this parameter\n        is associated\n    full_name : str\n        The name of the flow and parameter combined\n    parameter_name : str\n        The name of the parameter\n    data_type : str\n        The datatype of the parameter. generally unused for sklearn flows\n    default_value : str\n        The default value. For sklearn parameters, this is unknown and a\n        default value is selected arbitrarily\n    value : str\n        If the parameter was set, the value that it was set to.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        input_id: int,\n        flow_id: int,\n        flow_name: str,\n        full_name: str,\n        parameter_name: str,\n        data_type: str,\n        default_value: str,\n        value: str,\n    ):\n        self.id = input_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.full_name = full_name\n        self.parameter_name = parameter_name\n        self.data_type = data_type\n        self.default_value = default_value\n        self.value = value\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Parameter\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"ID\": self.id,\n            \"Flow ID\": self.flow_id,\n            # \"Flow Name\": self.flow_name,\n            \"Flow Name\": self.full_name,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Parameter Name\": self.parameter_name,\n        }\n        # indented prints for parameter attributes\n        # indention = 2 spaces + 1 | + 2 underscores\n        indent = \"{}|{}\".format(\" \" * 2, \"_\" * 2)\n        parameter_data_type = f\"{indent}Data Type\"\n        fields[parameter_data_type] = self.data_type\n        parameter_default = f\"{indent}Default\"\n        fields[parameter_default] = self.default_value\n        parameter_value = f\"{indent}Value\"\n        fields[parameter_value] = self.value\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Parameter Name\",\n            parameter_data_type,\n            parameter_default,\n            parameter_value,\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/setups/#openml.setups.OpenMLSetup","title":"<code>OpenMLSetup</code>","text":"<p>Setup object (a.k.a. Configuration).</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The OpenML setup id</p> required <code>flow_id</code> <code>int</code> <p>The flow that it is build upon</p> required <code>parameters</code> <code>dict</code> <p>The setting of the parameters</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLSetup:\n    \"\"\"Setup object (a.k.a. Configuration).\n\n    Parameters\n    ----------\n    setup_id : int\n        The OpenML setup id\n    flow_id : int\n        The flow that it is build upon\n    parameters : dict\n        The setting of the parameters\n    \"\"\"\n\n    def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None):\n        if not isinstance(setup_id, int):\n            raise ValueError(\"setup id should be int\")\n\n        if not isinstance(flow_id, int):\n            raise ValueError(\"flow id should be int\")\n\n        if parameters is not None and not isinstance(parameters, dict):\n            raise ValueError(\"parameters should be dict\")\n\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.parameters = parameters\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Setup\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Setup ID\": self.setup_id,\n            \"Flow ID\": self.flow_id,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"# of Parameters\": (\n                len(self.parameters) if self.parameters is not None else float(\"nan\")\n            ),\n        }\n\n        # determines the order in which the information will be printed\n        order = [\"Setup ID\", \"Flow ID\", \"Flow URL\", \"# of Parameters\"]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/setups/#openml.setups.get_setup","title":"<code>get_setup(setup_id)</code>","text":"<p>Downloads the setup (configuration) description from OpenML  and returns a structured object</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>OpenMLSetup (an initialized openml setup object)</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def get_setup(setup_id: int) -&gt; OpenMLSetup:\n    \"\"\"\n     Downloads the setup (configuration) description from OpenML\n     and returns a structured object\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    OpenMLSetup (an initialized openml setup object)\n    \"\"\"\n    setup_dir = Path(config.get_cache_directory()) / \"setups\" / str(setup_id)\n    setup_dir.mkdir(exist_ok=True, parents=True)\n\n    setup_file = setup_dir / \"description.xml\"\n\n    try:\n        return _get_cached_setup(setup_id)\n    except openml.exceptions.OpenMLCacheException:\n        url_suffix = \"/setup/%d\" % setup_id\n        setup_xml = openml._api_calls._perform_api_call(url_suffix, \"get\")\n        with setup_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(setup_xml)\n\n    result_dict = xmltodict.parse(setup_xml)\n    return _create_setup_from_xml(result_dict, output_format=\"object\")  # type: ignore\n</code></pre>"},{"location":"reference/setups/#openml.setups.initialize_model","title":"<code>initialize_model(setup_id)</code>","text":"<p>Initialized a model based on a setup_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def initialize_model(setup_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a setup_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    setup = get_setup(setup_id)\n    flow = openml.flows.get_flow(setup.flow_id)\n\n    # instead of using scikit-learns or any other library's \"set_params\" function, we override the\n    # OpenMLFlow objects default parameter value so we can utilize the\n    # Extension.flow_to_model() function to reinitialize the flow with the set defaults.\n    if setup.parameters is not None:\n        for hyperparameter in setup.parameters.values():\n            structure = flow.get_structure(\"flow_id\")\n            if len(structure[hyperparameter.flow_id]) &gt; 0:\n                subflow = flow.get_subflow(structure[hyperparameter.flow_id])\n            else:\n                subflow = flow\n            subflow.parameters[hyperparameter.parameter_name] = hyperparameter.value\n\n    return flow.extension.flow_to_model(flow)\n</code></pre>"},{"location":"reference/setups/#openml.setups.list_setups","title":"<code>list_setups(offset=None, size=None, flow=None, tag=None, setup=None, output_format='object')</code>","text":"<p>List all setups matching all of the given filters.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <code>None</code> <code>size</code> <code>int</code> <code>None</code> <code>flow</code> <code>int</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>setup</code> <code>Iterable[int]</code> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def list_setups(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    flow: int | None = None,\n    tag: str | None = None,\n    setup: Iterable[int] | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all setups matching all of the given filters.\n\n    Parameters\n    ----------\n    offset : int, optional\n    size : int, optional\n    flow : int, optional\n    tag : str, optional\n    setup : Iterable[int], optional\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict', 'object', or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    batch_size = 1000  # batch size for setups is lower\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_setups,\n        offset=offset,\n        size=size,\n        flow=flow,\n        tag=tag,\n        setup=setup,\n        batch_size=batch_size,\n    )\n</code></pre>"},{"location":"reference/setups/#openml.setups.setup_exists","title":"<code>setup_exists(flow)</code>","text":"<p>Checks whether a hyperparameter configuration already exists on the server.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>The openml flow object. Should have flow id present for the main flow and all subflows (i.e., it should be downloaded from the server by means of flow.get, and not instantiated locally)</p> required <p>Returns:</p> Name Type Description <code>setup_id</code> <code>int</code> <p>setup id iff exists, False otherwise</p> Source code in <code>openml/setups/functions.py</code> <pre><code>def setup_exists(flow: OpenMLFlow) -&gt; int:\n    \"\"\"\n    Checks whether a hyperparameter configuration already exists on the server.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        The openml flow object. Should have flow id present for the main flow\n        and all subflows (i.e., it should be downloaded from the server by\n        means of flow.get, and not instantiated locally)\n\n    Returns\n    -------\n    setup_id : int\n        setup id iff exists, False otherwise\n    \"\"\"\n    # sadly, this api call relies on a run object\n    openml.flows.functions._check_flow_for_server_id(flow)\n    if flow.model is None:\n        raise ValueError(\"Flow should have model field set with the actual model.\")\n    if flow.extension is None:\n        raise ValueError(\"Flow should have model field set with the correct extension.\")\n\n    # checks whether the flow exists on the server and flow ids align\n    exists = flow_exists(flow.name, flow.external_version)\n    if exists != flow.flow_id:\n        raise ValueError(\n            f\"Local flow id ({flow.id}) differs from server id ({exists}). \"\n            \"If this issue persists, please contact the developers.\",\n        )\n\n    openml_param_settings = flow.extension.obtain_parameter_values(flow)\n    description = xmltodict.unparse(_to_dict(flow.flow_id, openml_param_settings), pretty=True)\n    file_elements = {\n        \"description\": (\"description.arff\", description),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result = openml._api_calls._perform_api_call(\n        \"/setup/exists/\",\n        \"post\",\n        file_elements=file_elements,\n    )\n    result_dict = xmltodict.parse(result)\n    setup_id = int(result_dict[\"oml:setup_exists\"][\"oml:id\"])\n    return setup_id if setup_id &gt; 0 else False\n</code></pre>"},{"location":"reference/setups/functions/","title":"functions","text":""},{"location":"reference/setups/functions/#openml.setups.functions.__list_setups","title":"<code>__list_setups(api_call, output_format='object')</code>","text":"<p>Helper function to parse API calls which are lists of setups</p> Source code in <code>openml/setups/functions.py</code> <pre><code>def __list_setups(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\", \"object\"] = \"object\"\n) -&gt; dict[int, dict] | pd.DataFrame | dict[int, OpenMLSetup]:\n    \"\"\"Helper function to parse API calls which are lists of setups\"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    setups_dict = xmltodict.parse(xml_string, force_list=(\"oml:setup\",))\n    openml_uri = \"http://openml.org/openml\"\n    # Minimalistic check if the XML is useful\n    if \"oml:setups\" not in setups_dict:\n        raise ValueError(\n            'Error in return XML, does not contain \"oml:setups\":' \" %s\" % str(setups_dict),\n        )\n\n    if \"@xmlns:oml\" not in setups_dict[\"oml:setups\"]:\n        raise ValueError(\n            \"Error in return XML, does not contain \"\n            '\"oml:setups\"/@xmlns:oml: %s' % str(setups_dict),\n        )\n\n    if setups_dict[\"oml:setups\"][\"@xmlns:oml\"] != openml_uri:\n        raise ValueError(\n            \"Error in return XML, value of  \"\n            '\"oml:seyups\"/@xmlns:oml is not '\n            f'\"{openml_uri}\": {setups_dict!s}',\n        )\n\n    assert isinstance(setups_dict[\"oml:setups\"][\"oml:setup\"], list), type(setups_dict[\"oml:setups\"])\n\n    setups = {}\n    for setup_ in setups_dict[\"oml:setups\"][\"oml:setup\"]:\n        # making it a dict to give it the right format\n        current = _create_setup_from_xml(\n            {\"oml:setup_parameters\": setup_},\n            output_format=output_format,\n        )\n        if output_format == \"object\":\n            setups[current.setup_id] = current  # type: ignore\n        else:\n            setups[current[\"setup_id\"]] = current  # type: ignore\n\n    if output_format == \"dataframe\":\n        setups = pd.DataFrame.from_dict(setups, orient=\"index\")\n\n    return setups\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.get_setup","title":"<code>get_setup(setup_id)</code>","text":"<p>Downloads the setup (configuration) description from OpenML  and returns a structured object</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>OpenMLSetup (an initialized openml setup object)</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def get_setup(setup_id: int) -&gt; OpenMLSetup:\n    \"\"\"\n     Downloads the setup (configuration) description from OpenML\n     and returns a structured object\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    OpenMLSetup (an initialized openml setup object)\n    \"\"\"\n    setup_dir = Path(config.get_cache_directory()) / \"setups\" / str(setup_id)\n    setup_dir.mkdir(exist_ok=True, parents=True)\n\n    setup_file = setup_dir / \"description.xml\"\n\n    try:\n        return _get_cached_setup(setup_id)\n    except openml.exceptions.OpenMLCacheException:\n        url_suffix = \"/setup/%d\" % setup_id\n        setup_xml = openml._api_calls._perform_api_call(url_suffix, \"get\")\n        with setup_file.open(\"w\", encoding=\"utf8\") as fh:\n            fh.write(setup_xml)\n\n    result_dict = xmltodict.parse(setup_xml)\n    return _create_setup_from_xml(result_dict, output_format=\"object\")  # type: ignore\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.initialize_model","title":"<code>initialize_model(setup_id)</code>","text":"<p>Initialized a model based on a setup_id (i.e., using the exact same parameter settings)</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The Openml setup_id</p> required <p>Returns:</p> Type Description <code>model</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def initialize_model(setup_id: int) -&gt; Any:\n    \"\"\"\n    Initialized a model based on a setup_id (i.e., using the exact\n    same parameter settings)\n\n    Parameters\n    ----------\n    setup_id : int\n        The Openml setup_id\n\n    Returns\n    -------\n    model\n    \"\"\"\n    setup = get_setup(setup_id)\n    flow = openml.flows.get_flow(setup.flow_id)\n\n    # instead of using scikit-learns or any other library's \"set_params\" function, we override the\n    # OpenMLFlow objects default parameter value so we can utilize the\n    # Extension.flow_to_model() function to reinitialize the flow with the set defaults.\n    if setup.parameters is not None:\n        for hyperparameter in setup.parameters.values():\n            structure = flow.get_structure(\"flow_id\")\n            if len(structure[hyperparameter.flow_id]) &gt; 0:\n                subflow = flow.get_subflow(structure[hyperparameter.flow_id])\n            else:\n                subflow = flow\n            subflow.parameters[hyperparameter.parameter_name] = hyperparameter.value\n\n    return flow.extension.flow_to_model(flow)\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.list_setups","title":"<code>list_setups(offset=None, size=None, flow=None, tag=None, setup=None, output_format='object')</code>","text":"<p>List all setups matching all of the given filters.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <code>None</code> <code>size</code> <code>int</code> <code>None</code> <code>flow</code> <code>int</code> <code>None</code> <code>tag</code> <code>str</code> <code>None</code> <code>setup</code> <code>Iterable[int]</code> <code>None</code> <code>output_format</code> <code>Literal['object', 'dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'object'</code> <p>Returns:</p> Type Description <code>dict or dataframe</code> Source code in <code>openml/setups/functions.py</code> <pre><code>def list_setups(  # noqa: PLR0913\n    offset: int | None = None,\n    size: int | None = None,\n    flow: int | None = None,\n    tag: str | None = None,\n    setup: Iterable[int] | None = None,\n    output_format: Literal[\"object\", \"dict\", \"dataframe\"] = \"object\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    List all setups matching all of the given filters.\n\n    Parameters\n    ----------\n    offset : int, optional\n    size : int, optional\n    flow : int, optional\n    tag : str, optional\n    setup : Iterable[int], optional\n    output_format: str, optional (default='object')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    dict or dataframe\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\", \"object\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict', 'object', or 'dataframe' applicable.\",\n        )\n\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15. \"\n            \"To ensure your code will continue to work, \"\n            \"use `output_format`='dataframe' or `output_format`='object'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    batch_size = 1000  # batch size for setups is lower\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_setups,\n        offset=offset,\n        size=size,\n        flow=flow,\n        tag=tag,\n        setup=setup,\n        batch_size=batch_size,\n    )\n</code></pre>"},{"location":"reference/setups/functions/#openml.setups.functions.setup_exists","title":"<code>setup_exists(flow)</code>","text":"<p>Checks whether a hyperparameter configuration already exists on the server.</p> <p>Parameters:</p> Name Type Description Default <code>flow</code> <code>OpenMLFlow</code> <p>The openml flow object. Should have flow id present for the main flow and all subflows (i.e., it should be downloaded from the server by means of flow.get, and not instantiated locally)</p> required <p>Returns:</p> Name Type Description <code>setup_id</code> <code>int</code> <p>setup id iff exists, False otherwise</p> Source code in <code>openml/setups/functions.py</code> <pre><code>def setup_exists(flow: OpenMLFlow) -&gt; int:\n    \"\"\"\n    Checks whether a hyperparameter configuration already exists on the server.\n\n    Parameters\n    ----------\n    flow : OpenMLFlow\n        The openml flow object. Should have flow id present for the main flow\n        and all subflows (i.e., it should be downloaded from the server by\n        means of flow.get, and not instantiated locally)\n\n    Returns\n    -------\n    setup_id : int\n        setup id iff exists, False otherwise\n    \"\"\"\n    # sadly, this api call relies on a run object\n    openml.flows.functions._check_flow_for_server_id(flow)\n    if flow.model is None:\n        raise ValueError(\"Flow should have model field set with the actual model.\")\n    if flow.extension is None:\n        raise ValueError(\"Flow should have model field set with the correct extension.\")\n\n    # checks whether the flow exists on the server and flow ids align\n    exists = flow_exists(flow.name, flow.external_version)\n    if exists != flow.flow_id:\n        raise ValueError(\n            f\"Local flow id ({flow.id}) differs from server id ({exists}). \"\n            \"If this issue persists, please contact the developers.\",\n        )\n\n    openml_param_settings = flow.extension.obtain_parameter_values(flow)\n    description = xmltodict.unparse(_to_dict(flow.flow_id, openml_param_settings), pretty=True)\n    file_elements = {\n        \"description\": (\"description.arff\", description),\n    }  # type: openml._api_calls.FILE_ELEMENTS_TYPE\n    result = openml._api_calls._perform_api_call(\n        \"/setup/exists/\",\n        \"post\",\n        file_elements=file_elements,\n    )\n    result_dict = xmltodict.parse(result)\n    setup_id = int(result_dict[\"oml:setup_exists\"][\"oml:id\"])\n    return setup_id if setup_id &gt; 0 else False\n</code></pre>"},{"location":"reference/setups/setup/","title":"setup","text":""},{"location":"reference/setups/setup/#openml.setups.setup.OpenMLParameter","title":"<code>OpenMLParameter</code>","text":"<p>Parameter object (used in setup).</p> <p>Parameters:</p> Name Type Description Default <code>input_id</code> <code>int</code> <p>The input id from the openml database</p> required <code>flow</code> <p>The flow to which this parameter is associated</p> required <code>flow</code> <p>The name of the flow (no version number) to which this parameter is associated</p> required <code>full_name</code> <code>str</code> <p>The name of the flow and parameter combined</p> required <code>parameter_name</code> <code>str</code> <p>The name of the parameter</p> required <code>data_type</code> <code>str</code> <p>The datatype of the parameter. generally unused for sklearn flows</p> required <code>default_value</code> <code>str</code> <p>The default value. For sklearn parameters, this is unknown and a default value is selected arbitrarily</p> required <code>value</code> <code>str</code> <p>If the parameter was set, the value that it was set to.</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLParameter:\n    \"\"\"Parameter object (used in setup).\n\n    Parameters\n    ----------\n    input_id : int\n        The input id from the openml database\n    flow id : int\n        The flow to which this parameter is associated\n    flow name : str\n        The name of the flow (no version number) to which this parameter\n        is associated\n    full_name : str\n        The name of the flow and parameter combined\n    parameter_name : str\n        The name of the parameter\n    data_type : str\n        The datatype of the parameter. generally unused for sklearn flows\n    default_value : str\n        The default value. For sklearn parameters, this is unknown and a\n        default value is selected arbitrarily\n    value : str\n        If the parameter was set, the value that it was set to.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        input_id: int,\n        flow_id: int,\n        flow_name: str,\n        full_name: str,\n        parameter_name: str,\n        data_type: str,\n        default_value: str,\n        value: str,\n    ):\n        self.id = input_id\n        self.flow_id = flow_id\n        self.flow_name = flow_name\n        self.full_name = full_name\n        self.parameter_name = parameter_name\n        self.data_type = data_type\n        self.default_value = default_value\n        self.value = value\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Parameter\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"ID\": self.id,\n            \"Flow ID\": self.flow_id,\n            # \"Flow Name\": self.flow_name,\n            \"Flow Name\": self.full_name,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"Parameter Name\": self.parameter_name,\n        }\n        # indented prints for parameter attributes\n        # indention = 2 spaces + 1 | + 2 underscores\n        indent = \"{}|{}\".format(\" \" * 2, \"_\" * 2)\n        parameter_data_type = f\"{indent}Data Type\"\n        fields[parameter_data_type] = self.data_type\n        parameter_default = f\"{indent}Default\"\n        fields[parameter_default] = self.default_value\n        parameter_value = f\"{indent}Value\"\n        fields[parameter_value] = self.value\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Flow ID\",\n            \"Flow Name\",\n            \"Flow URL\",\n            \"Parameter Name\",\n            parameter_data_type,\n            parameter_default,\n            parameter_value,\n        ]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/setups/setup/#openml.setups.setup.OpenMLSetup","title":"<code>OpenMLSetup</code>","text":"<p>Setup object (a.k.a. Configuration).</p> <p>Parameters:</p> Name Type Description Default <code>setup_id</code> <code>int</code> <p>The OpenML setup id</p> required <code>flow_id</code> <code>int</code> <p>The flow that it is build upon</p> required <code>parameters</code> <code>dict</code> <p>The setting of the parameters</p> required Source code in <code>openml/setups/setup.py</code> <pre><code>class OpenMLSetup:\n    \"\"\"Setup object (a.k.a. Configuration).\n\n    Parameters\n    ----------\n    setup_id : int\n        The OpenML setup id\n    flow_id : int\n        The flow that it is build upon\n    parameters : dict\n        The setting of the parameters\n    \"\"\"\n\n    def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None):\n        if not isinstance(setup_id, int):\n            raise ValueError(\"setup id should be int\")\n\n        if not isinstance(flow_id, int):\n            raise ValueError(\"flow id should be int\")\n\n        if parameters is not None and not isinstance(parameters, dict):\n            raise ValueError(\"parameters should be dict\")\n\n        self.setup_id = setup_id\n        self.flow_id = flow_id\n        self.parameters = parameters\n\n    def __repr__(self) -&gt; str:\n        header = \"OpenML Setup\"\n        header = \"{}\\n{}\\n\".format(header, \"=\" * len(header))\n\n        fields = {\n            \"Setup ID\": self.setup_id,\n            \"Flow ID\": self.flow_id,\n            \"Flow URL\": openml.flows.OpenMLFlow.url_for_id(self.flow_id),\n            \"# of Parameters\": (\n                len(self.parameters) if self.parameters is not None else float(\"nan\")\n            ),\n        }\n\n        # determines the order in which the information will be printed\n        order = [\"Setup ID\", \"Flow ID\", \"Flow URL\", \"# of Parameters\"]\n        _fields = [(key, fields[key]) for key in order if key in fields]\n\n        longest_field_name_length = max(len(name) for name, _ in _fields)\n        field_line_format = f\"{{:.&lt;{longest_field_name_length}}}: {{}}\"\n        body = \"\\n\".join(field_line_format.format(name, value) for name, value in _fields)\n        return header + body\n</code></pre>"},{"location":"reference/study/","title":"study","text":""},{"location":"reference/study/#openml.study.OpenMLBenchmarkSuite","title":"<code>OpenMLBenchmarkSuite</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).</p> <p>It contains the following information: name, id, description, creation date, creator id and the task ids.</p> <p>According to this list of task ids, the suite object receives a list of OpenML object ids (datasets).</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLBenchmarkSuite(BaseStudy):\n    \"\"\"\n    An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and the task ids.\n\n    According to this list of task ids, the suite object receives a list of\n    OpenML object ids (datasets).\n\n    Parameters\n    ----------\n    suite_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        suite_id: int | None,\n        alias: str | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n    ):\n        super().__init__(\n            study_id=suite_id,\n            alias=alias,\n            main_entity_type=\"task\",\n            benchmark_suite=None,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=None,\n            runs=None,\n            setups=None,\n        )\n</code></pre>"},{"location":"reference/study/#openml.study.OpenMLStudy","title":"<code>OpenMLStudy</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLStudy represents the OpenML concept of a study (a collection of runs).</p> <p>It contains the following information: name, id, description, creation date, creator id and a list of run ids.</p> <p>According to this list of run ids, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLStudy(BaseStudy):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study (a collection of runs).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and a list of run ids.\n\n    According to this list of run ids, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        super().__init__(\n            study_id=study_id,\n            alias=alias,\n            main_entity_type=\"run\",\n            benchmark_suite=benchmark_suite,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=flows,\n            runs=runs,\n            setups=setups,\n        )\n</code></pre>"},{"location":"reference/study/#openml.study.attach_to_study","title":"<code>attach_to_study(study_id, run_ids)</code>","text":"<p>Attaches a set of runs to a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of runs to a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    result_xml = openml._api_calls._perform_api_call(\n        call=f\"study/{study_id}/attach\",\n        request_method=\"post\",\n        data={\"ids\": \",\".join(str(x) for x in run_ids)},\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_attach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/#openml.study.attach_to_suite","title":"<code>attach_to_suite(suite_id, task_ids)</code>","text":"<p>Attaches a set of tasks to a benchmarking suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the suite (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of tasks to a benchmarking suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the suite (in terms of explicitly linked entities)\n    \"\"\"\n    return attach_to_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/#openml.study.create_benchmark_suite","title":"<code>create_benchmark_suite(name, description, task_ids, alias=None)</code>","text":"<p>Creates an OpenML benchmark suite (collection of entity types, where the tasks are the linked entity)</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>task_ids</code> <code>list</code> <p>a list of task ids associated with this study more can be added later with <code>attach_to_suite</code>.</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_benchmark_suite(\n    name: str,\n    description: str,\n    task_ids: list[int],\n    alias: str | None = None,\n) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Creates an OpenML benchmark suite (collection of entity types, where\n    the tasks are the linked entity)\n\n    Parameters\n    ----------\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    task_ids : list\n        a list of task ids associated with this study\n        more can be added later with ``attach_to_suite``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLBenchmarkSuite(\n        suite_id=None,\n        alias=alias,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=task_ids,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.create_study","title":"<code>create_study(name, description, run_ids=None, alias=None, benchmark_suite=None)</code>","text":"<p>Creates an OpenML study (collection of data, tasks, flows, setups and run), where the runs are the main entity (collection consists of runs and all entities (flows, tasks, etc) that are related to these runs)</p> <p>Parameters:</p> Name Type Description Default <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran.</p> <code>None</code> <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>run_ids</code> <code>list</code> <p>a list of run ids associated with this study, these can also be added later with <code>attach_to_study</code>.</p> <code>None</code> <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <code>benchmark_suite</code> <code>int | None</code> <p>the ID of the suite for which this study contains run results</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_study(\n    name: str,\n    description: str,\n    run_ids: list[int] | None = None,\n    alias: str | None = None,\n    benchmark_suite: int | None = None,\n) -&gt; OpenMLStudy:\n    \"\"\"\n    Creates an OpenML study (collection of data, tasks, flows, setups and run),\n    where the runs are the main entity (collection consists of runs and all\n    entities (flows, tasks, etc) that are related to these runs)\n\n    Parameters\n    ----------\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    run_ids : list, optional\n        a list of run ids associated with this study,\n        these can also be added later with ``attach_to_study``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite: int (optional)\n        the ID of the suite for which this study contains run results\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLStudy(\n        study_id=None,\n        alias=alias,\n        benchmark_suite=benchmark_suite,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=None,\n        flows=None,\n        runs=run_ids if run_ids != [] else None,\n        setups=None,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.delete_study","title":"<code>delete_study(study_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_study(study_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return openml.utils._delete_entity(\"study\", study_id)\n</code></pre>"},{"location":"reference/study/#openml.study.delete_suite","title":"<code>delete_suite(suite_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_suite(suite_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return delete_study(suite_id)\n</code></pre>"},{"location":"reference/study/#openml.study.detach_from_study","title":"<code>detach_from_study(study_id, run_ids)</code>","text":"<p>Detaches a set of run ids from a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of run ids from a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    uri = \"study/%d/detach\" % study_id\n    post_variables = {\"ids\": \",\".join(str(x) for x in run_ids)}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        call=uri,\n        request_method=\"post\",\n        data=post_variables,\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_detach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/#openml.study.detach_from_suite","title":"<code>detach_from_suite(suite_id, task_ids)</code>","text":"<p>Detaches a set of task ids from a suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <code>new size of the study (in terms of explicitly linked entities)</code> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of task ids from a suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n    new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    return detach_from_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/#openml.study.get_study","title":"<code>get_study(study_id, arg_for_backwards_compat=None)</code>","text":"<p>Retrieves all relevant information of an OpenML study from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <code>arg_for_backwards_compat</code> <code>str</code> <p>The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the API which required specifying the type of study, i.e. tasks. We changed the implementation of studies since then and split them up into suites (collections of tasks) and studies (collections of runs) so this argument is no longer needed.</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>The OpenML study object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_study(\n    study_id: int | str,\n    arg_for_backwards_compat: str | None = None,  # noqa: ARG001\n) -&gt; OpenMLStudy:  # F401\n    \"\"\"\n    Retrieves all relevant information of an OpenML study from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    arg_for_backwards_compat : str, optional\n        The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the\n        API which required specifying the type of study, i.e. tasks. We changed the\n        implementation of studies since then and split them up into suites (collections of tasks)\n        and studies (collections of runs) so this argument is no longer needed.\n\n    Returns\n    -------\n    OpenMLStudy\n        The OpenML study object\n    \"\"\"\n    if study_id == \"OpenML100\":\n        message = (\n            \"It looks like you are running code from the OpenML100 paper. It still works, but lots \"\n            \"of things have changed since then. Please use `get_suite('OpenML100')` instead.\"\n        )\n        warnings.warn(message, DeprecationWarning, stacklevel=2)\n        openml.config.logger.warning(message)\n        study = _get_study(study_id, entity_type=\"task\")\n        assert isinstance(study, OpenMLBenchmarkSuite)\n\n        return study  # type: ignore\n\n    study = _get_study(study_id, entity_type=\"run\")\n    assert isinstance(study, OpenMLStudy)\n    return study\n</code></pre>"},{"location":"reference/study/#openml.study.get_suite","title":"<code>get_suite(suite_id)</code>","text":"<p>Retrieves all relevant information of an OpenML benchmarking suite from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <p>Returns:</p> Type Description <code>OpenMLSuite</code> <p>The OpenML suite object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_suite(suite_id: int | str) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Retrieves all relevant information of an OpenML benchmarking suite from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    Returns\n    -------\n    OpenMLSuite\n        The OpenML suite object\n    \"\"\"\n    study = _get_study(suite_id, entity_type=\"task\")\n    assert isinstance(study, OpenMLBenchmarkSuite)\n\n    return study\n</code></pre>"},{"location":"reference/study/#openml.study.list_studies","title":"<code>list_studies(offset=None, size=None, status=None, uploader=None, benchmark_suite=None, output_format='dict')</code>","text":"<pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all studies which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of studies to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of studies to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active studies are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return studies created by these users.</p> <code>None</code> <code>benchmark_suite</code> <code>int</code> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> <li> <p>If output_format='dataframe'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_studies(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[str] | None = None,\n    benchmark_suite: int | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all studies which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of studies to skip, starting from the first.\n    size : int, optional\n        The maximum number of studies to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        studies are returned.\n    uploader : list (int), optional\n        Result filter. Will only return studies created by these users.\n    benchmark_suite : int, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"run\",\n        status=status,\n        uploader=uploader,\n        benchmark_suite=benchmark_suite,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.list_suites","title":"<code>list_suites(offset=None, size=None, status=None, uploader=None, output_format='dict')</code>","text":"<pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all suites which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of suites to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of suites to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active suites are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return suites created by these users.</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every suite is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> <li> <p>If output_format='dataframe'     Every row is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_suites(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[int] | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all suites which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of suites to skip, starting from the first.\n    size : int, optional\n        The maximum number of suites to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        suites are returned.\n    uploader : list (int), optional\n        Result filter. Will only return suites created by these users.\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every suite is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n\n        - If output_format='dataframe'\n            Every row is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"task\",\n        status=status,\n        uploader=uploader,\n    )\n</code></pre>"},{"location":"reference/study/#openml.study.update_study_status","title":"<code>update_study_status(study_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_study_status(study_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    study_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(\"Illegal status value. \" \"Legal values: %s\" % legal_status)\n    data = {\"study_id\": study_id, \"status\": status}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"study/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_study_id = result[\"oml:study_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:study_status_update\"][\"oml:status\"]\n    if status != server_status or int(study_id) != int(server_study_id):\n        # This should never happen\n        raise ValueError(\"Study id/status does not collide\")\n</code></pre>"},{"location":"reference/study/#openml.study.update_suite_status","title":"<code>update_suite_status(suite_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_suite_status(suite_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    suite_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    return update_study_status(suite_id, status)\n</code></pre>"},{"location":"reference/study/functions/","title":"functions","text":""},{"location":"reference/study/functions/#openml.study.functions.__list_studies","title":"<code>__list_studies(api_call, output_format='dict')</code>","text":"<pre><code>__list_studies(api_call: str, output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>__list_studies(api_call: str, output_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Retrieves the list of OpenML studies and returns it in a dictionary or a Pandas DataFrame.</p> <p>Parameters:</p> Name Type Description Default <code>api_call</code> <code>str</code> <p>The API call for retrieving the list of OpenML studies.</p> required <code>output_format</code> <code>str in {'dict', 'dataframe'}</code> <p>Format of the output, either 'object' for a dictionary or 'dataframe' for a Pandas DataFrame.</p> <code>'dict'</code> <p>Returns:</p> Type Description <code>Union[Dict, DataFrame]</code> <p>A dictionary or Pandas DataFrame of OpenML studies, depending on the value of 'output_format'.</p> Source code in <code>openml/study/functions.py</code> <pre><code>def __list_studies(\n    api_call: str, output_format: Literal[\"dict\", \"dataframe\"] = \"dict\"\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Retrieves the list of OpenML studies and\n    returns it in a dictionary or a Pandas DataFrame.\n\n    Parameters\n    ----------\n    api_call : str\n        The API call for retrieving the list of OpenML studies.\n    output_format : str in {\"dict\", \"dataframe\"}\n        Format of the output, either 'object' for a dictionary\n        or 'dataframe' for a Pandas DataFrame.\n\n    Returns\n    -------\n    Union[Dict, pd.DataFrame]\n        A dictionary or Pandas DataFrame of OpenML studies,\n        depending on the value of 'output_format'.\n    \"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    study_dict = xmltodict.parse(xml_string, force_list=(\"oml:study\",))\n\n    # Minimalistic check if the XML is useful\n    assert isinstance(study_dict[\"oml:study_list\"][\"oml:study\"], list), type(\n        study_dict[\"oml:study_list\"],\n    )\n    assert study_dict[\"oml:study_list\"][\"@xmlns:oml\"] == \"http://openml.org/openml\", study_dict[\n        \"oml:study_list\"\n    ][\"@xmlns:oml\"]\n\n    studies = {}\n    for study_ in study_dict[\"oml:study_list\"][\"oml:study\"]:\n        # maps from xml name to a tuple of (dict name, casting fn)\n        expected_fields = {\n            \"oml:id\": (\"id\", int),\n            \"oml:alias\": (\"alias\", str),\n            \"oml:main_entity_type\": (\"main_entity_type\", str),\n            \"oml:benchmark_suite\": (\"benchmark_suite\", int),\n            \"oml:name\": (\"name\", str),\n            \"oml:status\": (\"status\", str),\n            \"oml:creation_date\": (\"creation_date\", str),\n            \"oml:creator\": (\"creator\", int),\n        }\n        study_id = int(study_[\"oml:id\"])\n        current_study = {}\n        for oml_field_name, (real_field_name, cast_fn) in expected_fields.items():\n            if oml_field_name in study_:\n                current_study[real_field_name] = cast_fn(study_[oml_field_name])\n        current_study[\"id\"] = int(current_study[\"id\"])\n        studies[study_id] = current_study\n\n    if output_format == \"dataframe\":\n        studies = pd.DataFrame.from_dict(studies, orient=\"index\")\n    return studies\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.attach_to_study","title":"<code>attach_to_study(study_id, run_ids)</code>","text":"<p>Attaches a set of runs to a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of runs to a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    result_xml = openml._api_calls._perform_api_call(\n        call=f\"study/{study_id}/attach\",\n        request_method=\"post\",\n        data={\"ids\": \",\".join(str(x) for x in run_ids)},\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_attach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.attach_to_suite","title":"<code>attach_to_suite(suite_id, task_ids)</code>","text":"<p>Attaches a set of tasks to a benchmarking suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to link to the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the suite (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def attach_to_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Attaches a set of tasks to a benchmarking suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to link to the collection\n\n    Returns\n    -------\n    int\n        new size of the suite (in terms of explicitly linked entities)\n    \"\"\"\n    return attach_to_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.create_benchmark_suite","title":"<code>create_benchmark_suite(name, description, task_ids, alias=None)</code>","text":"<p>Creates an OpenML benchmark suite (collection of entity types, where the tasks are the linked entity)</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>task_ids</code> <code>list</code> <p>a list of task ids associated with this study more can be added later with <code>attach_to_suite</code>.</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_benchmark_suite(\n    name: str,\n    description: str,\n    task_ids: list[int],\n    alias: str | None = None,\n) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Creates an OpenML benchmark suite (collection of entity types, where\n    the tasks are the linked entity)\n\n    Parameters\n    ----------\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    task_ids : list\n        a list of task ids associated with this study\n        more can be added later with ``attach_to_suite``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLBenchmarkSuite(\n        suite_id=None,\n        alias=alias,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=task_ids,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.create_study","title":"<code>create_study(name, description, run_ids=None, alias=None, benchmark_suite=None)</code>","text":"<p>Creates an OpenML study (collection of data, tasks, flows, setups and run), where the runs are the main entity (collection consists of runs and all entities (flows, tasks, etc) that are related to these runs)</p> <p>Parameters:</p> Name Type Description Default <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran.</p> <code>None</code> <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>run_ids</code> <code>list</code> <p>a list of run ids associated with this study, these can also be added later with <code>attach_to_study</code>.</p> <code>None</code> <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> <code>None</code> <code>benchmark_suite</code> <code>int | None</code> <p>the ID of the suite for which this study contains run results</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>A local OpenML study object (call publish method to upload to server)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def create_study(\n    name: str,\n    description: str,\n    run_ids: list[int] | None = None,\n    alias: str | None = None,\n    benchmark_suite: int | None = None,\n) -&gt; OpenMLStudy:\n    \"\"\"\n    Creates an OpenML study (collection of data, tasks, flows, setups and run),\n    where the runs are the main entity (collection consists of runs and all\n    entities (flows, tasks, etc) that are related to these runs)\n\n    Parameters\n    ----------\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    run_ids : list, optional\n        a list of run ids associated with this study,\n        these can also be added later with ``attach_to_study``.\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite: int (optional)\n        the ID of the suite for which this study contains run results\n\n    Returns\n    -------\n    OpenMLStudy\n        A local OpenML study object (call publish method to upload to server)\n    \"\"\"\n    return OpenMLStudy(\n        study_id=None,\n        alias=alias,\n        benchmark_suite=benchmark_suite,\n        name=name,\n        description=description,\n        status=None,\n        creation_date=None,\n        creator=None,\n        tags=None,\n        data=None,\n        tasks=None,\n        flows=None,\n        runs=run_ids if run_ids != [] else None,\n        setups=None,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.delete_study","title":"<code>delete_study(study_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_study(study_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return openml.utils._delete_entity(\"study\", study_id)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.delete_suite","title":"<code>delete_suite(suite_id)</code>","text":"<p>Deletes a study from the OpenML server.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True iff the deletion was successful. False otherwise</p> Source code in <code>openml/study/functions.py</code> <pre><code>def delete_suite(suite_id: int) -&gt; bool:\n    \"\"\"Deletes a study from the OpenML server.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    Returns\n    -------\n    bool\n        True iff the deletion was successful. False otherwise\n    \"\"\"\n    return delete_study(suite_id)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.detach_from_study","title":"<code>detach_from_study(study_id, run_ids)</code>","text":"<p>Detaches a set of run ids from a study.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>run_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <p>new size of the study (in terms of explicitly linked entities)</p> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_study(study_id: int, run_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of run ids from a study.\n\n    Parameters\n    ----------\n    study_id : int\n        OpenML id of the study\n\n    run_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n        new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    # Interestingly, there's no need to tell the server about the entity type, it knows by itself\n    uri = \"study/%d/detach\" % study_id\n    post_variables = {\"ids\": \",\".join(str(x) for x in run_ids)}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\n        call=uri,\n        request_method=\"post\",\n        data=post_variables,\n    )\n    result = xmltodict.parse(result_xml)[\"oml:study_detach\"]\n    return int(result[\"oml:linked_entities\"])\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.detach_from_suite","title":"<code>detach_from_suite(suite_id, task_ids)</code>","text":"<p>Detaches a set of task ids from a suite.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>OpenML id of the study</p> required <code>task_ids</code> <code>list(int)</code> <p>List of entities to unlink from the collection</p> required <p>Returns:</p> Type Description <code>int</code> <code>new size of the study (in terms of explicitly linked entities)</code> Source code in <code>openml/study/functions.py</code> <pre><code>def detach_from_suite(suite_id: int, task_ids: list[int]) -&gt; int:\n    \"\"\"Detaches a set of task ids from a suite.\n\n    Parameters\n    ----------\n    suite_id : int\n        OpenML id of the study\n\n    task_ids : list (int)\n        List of entities to unlink from the collection\n\n    Returns\n    -------\n    int\n    new size of the study (in terms of explicitly linked entities)\n    \"\"\"\n    return detach_from_study(suite_id, task_ids)\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.get_study","title":"<code>get_study(study_id, arg_for_backwards_compat=None)</code>","text":"<p>Retrieves all relevant information of an OpenML study from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <code>arg_for_backwards_compat</code> <code>str</code> <p>The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the API which required specifying the type of study, i.e. tasks. We changed the implementation of studies since then and split them up into suites (collections of tasks) and studies (collections of runs) so this argument is no longer needed.</p> <code>None</code> <p>Returns:</p> Type Description <code>OpenMLStudy</code> <p>The OpenML study object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_study(\n    study_id: int | str,\n    arg_for_backwards_compat: str | None = None,  # noqa: ARG001\n) -&gt; OpenMLStudy:  # F401\n    \"\"\"\n    Retrieves all relevant information of an OpenML study from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    arg_for_backwards_compat : str, optional\n        The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the\n        API which required specifying the type of study, i.e. tasks. We changed the\n        implementation of studies since then and split them up into suites (collections of tasks)\n        and studies (collections of runs) so this argument is no longer needed.\n\n    Returns\n    -------\n    OpenMLStudy\n        The OpenML study object\n    \"\"\"\n    if study_id == \"OpenML100\":\n        message = (\n            \"It looks like you are running code from the OpenML100 paper. It still works, but lots \"\n            \"of things have changed since then. Please use `get_suite('OpenML100')` instead.\"\n        )\n        warnings.warn(message, DeprecationWarning, stacklevel=2)\n        openml.config.logger.warning(message)\n        study = _get_study(study_id, entity_type=\"task\")\n        assert isinstance(study, OpenMLBenchmarkSuite)\n\n        return study  # type: ignore\n\n    study = _get_study(study_id, entity_type=\"run\")\n    assert isinstance(study, OpenMLStudy)\n    return study\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.get_suite","title":"<code>get_suite(suite_id)</code>","text":"<p>Retrieves all relevant information of an OpenML benchmarking suite from the server.</p> <p>Parameters:</p> Name Type Description Default <code>study</code> <p>study id (numeric or alias)</p> required <p>Returns:</p> Type Description <code>OpenMLSuite</code> <p>The OpenML suite object</p> Source code in <code>openml/study/functions.py</code> <pre><code>def get_suite(suite_id: int | str) -&gt; OpenMLBenchmarkSuite:\n    \"\"\"\n    Retrieves all relevant information of an OpenML benchmarking suite from the server.\n\n    Parameters\n    ----------\n    study id : int, str\n        study id (numeric or alias)\n\n    Returns\n    -------\n    OpenMLSuite\n        The OpenML suite object\n    \"\"\"\n    study = _get_study(suite_id, entity_type=\"task\")\n    assert isinstance(study, OpenMLBenchmarkSuite)\n\n    return study\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.list_studies","title":"<code>list_studies(offset=None, size=None, status=None, uploader=None, benchmark_suite=None, output_format='dict')</code>","text":"<pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_studies(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[str] | None = ..., benchmark_suite: int | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all studies which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of studies to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of studies to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active studies are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return studies created by these users.</p> <code>None</code> <code>benchmark_suite</code> <code>int</code> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> <li> <p>If output_format='dataframe'     Every dataset is represented by a dictionary containing     the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>benchmark_suite (optional)</li> <li>status</li> <li>creator</li> <li>creation_date If qualities are calculated for the dataset, some of these are also returned.</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_studies(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[str] | None = None,\n    benchmark_suite: int | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all studies which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of studies to skip, starting from the first.\n    size : int, optional\n        The maximum number of studies to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        studies are returned.\n    uploader : list (int), optional\n        Result filter. Will only return studies created by these users.\n    benchmark_suite : int, optional\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n\n        - If output_format='dataframe'\n            Every dataset is represented by a dictionary containing\n            the following information:\n            - id\n            - alias (optional)\n            - name\n            - benchmark_suite (optional)\n            - status\n            - creator\n            - creation_date\n            If qualities are calculated for the dataset, some of\n            these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"run\",\n        status=status,\n        uploader=uploader,\n        benchmark_suite=benchmark_suite,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.list_suites","title":"<code>list_suites(offset=None, size=None, status=None, uploader=None, output_format='dict')</code>","text":"<pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dict'] = 'dict') -&gt; dict\n</code></pre><pre><code>list_suites(offset: int | None = ..., size: int | None = ..., status: str | None = ..., uploader: list[int] | None = ..., output_format: Literal['dataframe'] = 'dataframe') -&gt; pd.DataFrame\n</code></pre> <p>Return a list of all suites which are on OpenML.</p> <p>Parameters:</p> Name Type Description Default <code>offset</code> <code>int</code> <p>The number of suites to skip, starting from the first.</p> <code>None</code> <code>size</code> <code>int</code> <p>The maximum number of suites to show.</p> <code>None</code> <code>status</code> <code>str</code> <p>Should be {active, in_preparation, deactivated, all}. By default active suites are returned.</p> <code>None</code> <code>uploader</code> <code>list(int)</code> <p>Result filter. Will only return suites created by these users.</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <p>Returns:</p> Name Type Description <code>datasets</code> <code>dict of dicts, or dataframe</code> <ul> <li> <p>If output_format='dict'     Every suite is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> <li> <p>If output_format='dataframe'     Every row is represented by a dictionary containing the following information:</p> <ul> <li>id</li> <li>alias (optional)</li> <li>name</li> <li>main_entity_type</li> <li>status</li> <li>creator</li> <li>creation_date</li> </ul> </li> </ul> Source code in <code>openml/study/functions.py</code> <pre><code>def list_suites(\n    offset: int | None = None,\n    size: int | None = None,\n    status: str | None = None,\n    uploader: list[int] | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a list of all suites which are on OpenML.\n\n    Parameters\n    ----------\n    offset : int, optional\n        The number of suites to skip, starting from the first.\n    size : int, optional\n        The maximum number of suites to show.\n    status : str, optional\n        Should be {active, in_preparation, deactivated, all}. By default active\n        suites are returned.\n    uploader : list (int), optional\n        Result filter. Will only return suites created by these users.\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n\n    Returns\n    -------\n    datasets : dict of dicts, or dataframe\n        - If output_format='dict'\n            Every suite is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n\n        - If output_format='dataframe'\n            Every row is represented by a dictionary containing the following information:\n            - id\n            - alias (optional)\n            - name\n            - main_entity_type\n            - status\n            - creator\n            - creation_date\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_studies,\n        offset=offset,\n        size=size,\n        main_entity_type=\"task\",\n        status=status,\n        uploader=uploader,\n    )\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.update_study_status","title":"<code>update_study_status(study_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_study_status(study_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    study_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    legal_status = {\"active\", \"deactivated\"}\n    if status not in legal_status:\n        raise ValueError(\"Illegal status value. \" \"Legal values: %s\" % legal_status)\n    data = {\"study_id\": study_id, \"status\": status}  # type: openml._api_calls.DATA_TYPE\n    result_xml = openml._api_calls._perform_api_call(\"study/status/update\", \"post\", data=data)\n    result = xmltodict.parse(result_xml)\n    server_study_id = result[\"oml:study_status_update\"][\"oml:id\"]\n    server_status = result[\"oml:study_status_update\"][\"oml:status\"]\n    if status != server_status or int(study_id) != int(server_study_id):\n        # This should never happen\n        raise ValueError(\"Study id/status does not collide\")\n</code></pre>"},{"location":"reference/study/functions/#openml.study.functions.update_suite_status","title":"<code>update_suite_status(suite_id, status)</code>","text":"<p>Updates the status of a study to either 'active' or 'deactivated'.</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>The data id of the dataset</p> required <code>status</code> <code>(str)</code> <p>'active' or 'deactivated'</p> required Source code in <code>openml/study/functions.py</code> <pre><code>def update_suite_status(suite_id: int, status: str) -&gt; None:\n    \"\"\"\n    Updates the status of a study to either 'active' or 'deactivated'.\n\n    Parameters\n    ----------\n    suite_id : int\n        The data id of the dataset\n    status : str,\n        'active' or 'deactivated'\n    \"\"\"\n    return update_study_status(suite_id, status)\n</code></pre>"},{"location":"reference/study/study/","title":"study","text":""},{"location":"reference/study/study/#openml.study.study.BaseStudy","title":"<code>BaseStudy</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>An OpenMLStudy represents the OpenML concept of a study. It contains the following information: name, id, description, creation date, creator id and a set of tags.</p> <p>According to this list of tags, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Can be used to obtain all relevant information from a study at once.</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class BaseStudy(OpenMLBase):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study. It contains\n    the following information: name, id, description, creation date,\n    creator id and a set of tags.\n\n    According to this list of tags, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Can be used to obtain all relevant information from a study at once.\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        main_entity_type: str,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        self.study_id = study_id\n        self.alias = alias\n        self.main_entity_type = main_entity_type\n        self.benchmark_suite = benchmark_suite\n        self.name = name\n        self.description = description\n        self.status = status\n        self.creation_date = creation_date\n        self.creator = creator\n        self.tags = tags  # LEGACY. Can be removed soon\n        self.data = data\n        self.tasks = tasks\n        self.flows = flows\n        self.setups = setups\n        self.runs = runs\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"s\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the id of the study.\"\"\"\n        return self.study_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        fields: dict[str, Any] = {\n            \"Name\": self.name,\n            \"Status\": self.status,\n            \"Main Entity Type\": self.main_entity_type,\n        }\n        if self.study_id is not None:\n            fields[\"ID\"] = self.study_id\n            fields[\"Study URL\"] = self.openml_url\n        if self.creator is not None:\n            fields[\"Creator\"] = f\"{get_server_base_url()}/u/{self.creator}\"\n        if self.creation_date is not None:\n            fields[\"Upload Time\"] = self.creation_date.replace(\"T\", \" \")\n        if self.data is not None:\n            fields[\"# of Data\"] = len(self.data)\n        if self.tasks is not None:\n            fields[\"# of Tasks\"] = len(self.tasks)\n        if self.flows is not None:\n            fields[\"# of Flows\"] = len(self.flows)\n        if self.runs is not None:\n            fields[\"# of Runs\"] = len(self.runs)\n\n        # determines the order in which the information will be printed\n        order = [\n            \"ID\",\n            \"Name\",\n            \"Status\",\n            \"Main Entity Type\",\n            \"Study URL\",\n            \"# of Data\",\n            \"# of Tasks\",\n            \"# of Flows\",\n            \"# of Runs\",\n            \"Creator\",\n            \"Upload Time\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.study_id = int(xml_response[\"oml:study_upload\"][\"oml:id\"])\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        \"\"\"Creates a dictionary representation of self.\"\"\"\n        # some can not be uploaded, e.g., id, creator, creation_date\n        simple_props = [\"alias\", \"main_entity_type\", \"name\", \"description\"]\n\n        # TODO(eddiebergman): Begging for a walrus if we can drop 3.7\n        simple_prop_values = {}\n        for prop_name in simple_props:\n            content = getattr(self, prop_name, None)\n            if content is not None:\n                simple_prop_values[\"oml:\" + prop_name] = content\n\n        # maps from attribute name (which is used as outer tag name) to immer\n        # tag name e.g., self.tasks -&gt; &lt;oml:tasks&gt;&lt;oml:task_id&gt;1987&lt;/oml:task_id&gt;&lt;/oml:tasks&gt;\n        complex_props = {\"tasks\": \"task_id\", \"runs\": \"run_id\"}\n\n        # TODO(eddiebergman): Begging for a walrus if we can drop 3.7\n        complex_prop_values = {}\n        for prop_name, inner_name in complex_props.items():\n            content = getattr(self, prop_name, None)\n            if content is not None:\n                complex_prop_values[\"oml:\" + prop_name] = {\"oml:\" + inner_name: content}\n\n        return {\n            \"oml:study\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                **simple_prop_values,\n                **complex_prop_values,\n            }\n        }\n\n    def push_tag(self, tag: str) -&gt; None:\n        \"\"\"Add a tag to the study.\"\"\"\n        raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n\n    def remove_tag(self, tag: str) -&gt; None:\n        \"\"\"Remove a tag from the study.\"\"\"\n        raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.BaseStudy.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the id of the study.</p>"},{"location":"reference/study/study/#openml.study.study.BaseStudy.push_tag","title":"<code>push_tag(tag)</code>","text":"<p>Add a tag to the study.</p> Source code in <code>openml/study/study.py</code> <pre><code>def push_tag(self, tag: str) -&gt; None:\n    \"\"\"Add a tag to the study.\"\"\"\n    raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.BaseStudy.remove_tag","title":"<code>remove_tag(tag)</code>","text":"<p>Remove a tag from the study.</p> Source code in <code>openml/study/study.py</code> <pre><code>def remove_tag(self, tag: str) -&gt; None:\n    \"\"\"Remove a tag from the study.\"\"\"\n    raise NotImplementedError(\"Tags for studies is not (yet) supported.\")\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.OpenMLBenchmarkSuite","title":"<code>OpenMLBenchmarkSuite</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).</p> <p>It contains the following information: name, id, description, creation date, creator id and the task ids.</p> <p>According to this list of task ids, the suite object receives a list of OpenML object ids (datasets).</p> <p>Parameters:</p> Name Type Description Default <code>suite_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>main_entity_type</code> <code>str</code> <p>the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLBenchmarkSuite(BaseStudy):\n    \"\"\"\n    An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and the task ids.\n\n    According to this list of task ids, the suite object receives a list of\n    OpenML object ids (datasets).\n\n    Parameters\n    ----------\n    suite_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    main_entity_type : str\n        the entity type (e.g., task, run) that is core in this study.\n        only entities of this type can be added explicitly\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        suite_id: int | None,\n        alias: str | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n    ):\n        super().__init__(\n            study_id=suite_id,\n            alias=alias,\n            main_entity_type=\"task\",\n            benchmark_suite=None,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=None,\n            runs=None,\n            setups=None,\n        )\n</code></pre>"},{"location":"reference/study/study/#openml.study.study.OpenMLStudy","title":"<code>OpenMLStudy</code>","text":"<p>               Bases: <code>BaseStudy</code></p> <p>An OpenMLStudy represents the OpenML concept of a study (a collection of runs).</p> <p>It contains the following information: name, id, description, creation date, creator id and a list of run ids.</p> <p>According to this list of run ids, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).</p> <p>Parameters:</p> Name Type Description Default <code>study_id</code> <code>int</code> <p>the study id</p> required <code>alias</code> <code>str(optional)</code> <p>a string ID, unique on server (url-friendly)</p> required <code>benchmark_suite</code> <code>int(optional)</code> <p>the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs.</p> required <code>name</code> <code>str</code> <p>the name of the study (meta-info)</p> required <code>description</code> <code>str</code> <p>brief description (meta-info)</p> required <code>status</code> <code>str</code> <p>Whether the study is in preparation, active or deactivated</p> required <code>creation_date</code> <code>str</code> <p>date of creation (meta-info)</p> required <code>creator</code> <code>int</code> <p>openml user id of the owner / creator</p> required <code>tags</code> <code>list(dict)</code> <p>The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access.</p> required <code>data</code> <code>list</code> <p>a list of data ids associated with this study</p> required <code>tasks</code> <code>list</code> <p>a list of task ids associated with this study</p> required <code>flows</code> <code>list</code> <p>a list of flow ids associated with this study</p> required <code>runs</code> <code>list</code> <p>a list of run ids associated with this study</p> required <code>setups</code> <code>list</code> <p>a list of setup ids associated with this study</p> required Source code in <code>openml/study/study.py</code> <pre><code>class OpenMLStudy(BaseStudy):\n    \"\"\"\n    An OpenMLStudy represents the OpenML concept of a study (a collection of runs).\n\n    It contains the following information: name, id, description, creation date,\n    creator id and a list of run ids.\n\n    According to this list of run ids, the study object receives a list of\n    OpenML object ids (datasets, flows, tasks and setups).\n\n    Parameters\n    ----------\n    study_id : int\n        the study id\n    alias : str (optional)\n        a string ID, unique on server (url-friendly)\n    benchmark_suite : int (optional)\n        the benchmark suite (another study) upon which this study is ran.\n        can only be active if main entity type is runs.\n    name : str\n        the name of the study (meta-info)\n    description : str\n        brief description (meta-info)\n    status : str\n        Whether the study is in preparation, active or deactivated\n    creation_date : str\n        date of creation (meta-info)\n    creator : int\n        openml user id of the owner / creator\n    tags : list(dict)\n        The list of tags shows which tags are associated with the study.\n        Each tag is a dict of (tag) name, window_start and write_access.\n    data : list\n        a list of data ids associated with this study\n    tasks : list\n        a list of task ids associated with this study\n    flows : list\n        a list of flow ids associated with this study\n    runs : list\n        a list of run ids associated with this study\n    setups : list\n        a list of setup ids associated with this study\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        study_id: int | None,\n        alias: str | None,\n        benchmark_suite: int | None,\n        name: str,\n        description: str,\n        status: str | None,\n        creation_date: str | None,\n        creator: int | None,\n        tags: list[dict] | None,\n        data: list[int] | None,\n        tasks: list[int] | None,\n        flows: list[int] | None,\n        runs: list[int] | None,\n        setups: list[int] | None,\n    ):\n        super().__init__(\n            study_id=study_id,\n            alias=alias,\n            main_entity_type=\"run\",\n            benchmark_suite=benchmark_suite,\n            name=name,\n            description=description,\n            status=status,\n            creation_date=creation_date,\n            creator=creator,\n            tags=tags,\n            data=data,\n            tasks=tasks,\n            flows=flows,\n            runs=runs,\n            setups=setups,\n        )\n</code></pre>"},{"location":"reference/tasks/","title":"tasks","text":""},{"location":"reference/tasks/#openml.tasks.OpenMLClassificationTask","title":"<code>OpenMLClassificationTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Classification task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the Classification task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the Classification task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target variable.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the Classification task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the Classification task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the Classification task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Classification task (if it already exists on OpenML).</p> <code>None</code> <code>class_labels</code> <code>List of str</code> <p>A list of class labels (for classification tasks).</p> <code>None</code> <code>cost_matrix</code> <code>array</code> <p>A cost matrix (for classification tasks).</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClassificationTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Classification task type.\n    task_type : str\n        Name of the Classification task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the Classification task.\n    target_name : str\n        Name of the target variable.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the Classification task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the Classification task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure.\n    data_splits_url : str, default=None\n        URL of the data splits for the Classification task.\n    task_id : Union[int, None]\n        ID of the Classification task (if it already exists on OpenML).\n    class_labels : List of str, default=None\n        A list of class labels (for classification tasks).\n    cost_matrix : array, default=None\n        A cost matrix (for classification tasks).\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n        self.class_labels = class_labels\n        self.cost_matrix = cost_matrix\n\n        if cost_matrix is not None:\n            raise NotImplementedError(\"Costmatrix\")\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLClusteringTask","title":"<code>OpenMLClusteringTask</code>","text":"<p>               Bases: <code>OpenMLTask</code></p> <p>OpenML Clustering object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML clustering task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML clustering task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset used in clustering the task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML clustering task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure used in the clustering task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the clustering task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the clustering task.</p> <code>None</code> <code>target_name</code> <code>str</code> <p>Name of the target feature (class) that is not part of the feature set for the clustering task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClusteringTask(OpenMLTask):\n    \"\"\"OpenML Clustering object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML clustering task.\n    task_type : str\n        Task type of the OpenML clustering task.\n    data_set_id : int\n        ID of the OpenML dataset used in clustering the task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    task_id : Union[int, None]\n        ID of the OpenML clustering task.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure used in the clustering task.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the clustering task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the clustering task.\n    target_name : str, default=None\n        Name of the target feature (class) that is not part of the\n        feature set for the clustering task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 17,\n        task_id: int | None = None,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        evaluation_measure: str | None = None,\n        target_name: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            evaluation_measure=evaluation_measure,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\"] = \"array\",\n    ) -&gt; np.ndarray | scipy.sparse.spmatrix:\n        ...\n\n    @overload\n    def get_X(self, dataset_format: Literal[\"dataframe\"]) -&gt; pd.DataFrame:\n        ...\n\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n    ) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        dataset = self.get_dataset()\n        data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n        return data\n\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        # Right now, it is not supported as a feature.\n        # Uncomment if it is supported on the server\n        # in the future.\n        # https://github.com/openml/OpenML/issues/925\n        \"\"\"\n        task_dict = task_container['oml:task_inputs']\n        if self.target_name is not None:\n            task_dict['oml:input'].append(\n                OrderedDict([\n                    ('@name', 'target_feature'),\n                    ('#text', self.target_name)\n                ])\n            )\n        \"\"\"\n        return super()._to_dict()\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLClusteringTask.get_X","title":"<code>get_X(dataset_format='array')</code>","text":"<pre><code>get_X(dataset_format: Literal['array'] = 'array') -&gt; np.ndarray | scipy.sparse.spmatrix\n</code></pre><pre><code>get_X(dataset_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X(\n    self,\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    dataset = self.get_dataset()\n    data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n    return data\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLLearningCurveTask","title":"<code>OpenMLLearningCurveTask</code>","text":"<p>               Bases: <code>OpenMLClassificationTask</code></p> <p>OpenML Learning Curve object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Learning Curve task.</p> required <code>task_type</code> <code>str</code> <p>Name of the Learning Curve task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the dataset that this task is associated with.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature in the dataset.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure to use for evaluating models.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Additional parameters for the estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the file containing the data splits for Learning Curve task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Learning Curve task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure to use for evaluating models.</p> <code>None</code> <code>class_labels</code> <code>list of str</code> <p>Class labels for Learning Curve tasks.</p> <code>None</code> <code>cost_matrix</code> <code>numpy array</code> <p>Cost matrix for Learning Curve tasks.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLLearningCurveTask(OpenMLClassificationTask):\n    \"\"\"OpenML Learning Curve object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Learning Curve task.\n    task_type : str\n        Name of the Learning Curve task.\n    data_set_id : int\n        ID of the dataset that this task is associated with.\n    target_name : str\n        Name of the target feature in the dataset.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure to use for evaluating models.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Additional parameters for the estimation procedure.\n    data_splits_url : str, default=None\n        URL of the file containing the data splits for Learning Curve task.\n    task_id : Union[int, None]\n        ID of the Learning Curve task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure to use for evaluating models.\n    class_labels : list of str, default=None\n        Class labels for Learning Curve tasks.\n    cost_matrix : numpy array, default=None\n        Cost matrix for Learning Curve tasks.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 13,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n            class_labels=class_labels,\n            cost_matrix=cost_matrix,\n        )\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLRegressionTask","title":"<code>OpenMLRegressionTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Regression object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML Regression task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML Regression task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature used in the Regression task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the Regression task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML Regression task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the Regression task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLRegressionTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Regression object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML Regression task.\n    task_type : str\n        Task type of the OpenML Regression task.\n    data_set_id : int\n        ID of the OpenML dataset.\n    target_name : str\n        Name of the target feature used in the Regression task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the Regression task.\n    task_id : Union[int, None]\n        ID of the OpenML Regression task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the Regression task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 7,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSplit","title":"<code>OpenMLSplit</code>","text":"<p>OpenML Split object.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>int or str</code> required <code>description</code> <code>str</code> required <code>split</code> <code>dict</code> required Source code in <code>openml/tasks/split.py</code> <pre><code>class OpenMLSplit:\n    \"\"\"OpenML Split object.\n\n    Parameters\n    ----------\n    name : int or str\n    description : str\n    split : dict\n    \"\"\"\n\n    def __init__(\n        self,\n        name: int | str,\n        description: str,\n        split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]],\n    ):\n        self.description = description\n        self.name = name\n        self.split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]] = {}\n\n        # Add splits according to repetition\n        for repetition in split:\n            _rep = int(repetition)\n            self.split[_rep] = OrderedDict()\n            for fold in split[_rep]:\n                self.split[_rep][fold] = OrderedDict()\n                for sample in split[_rep][fold]:\n                    self.split[_rep][fold][sample] = split[_rep][fold][sample]\n\n        self.repeats = len(self.split)\n\n        # TODO(eddiebergman): Better error message\n        if any(len(self.split[0]) != len(self.split[i]) for i in range(self.repeats)):\n            raise ValueError(\"\")\n\n        self.folds = len(self.split[0])\n        self.samples = len(self.split[0][0])\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if (\n            (not isinstance(self, type(other)))\n            or self.name != other.name\n            or self.description != other.description\n            or self.split.keys() != other.split.keys()\n            or any(\n                self.split[repetition].keys() != other.split[repetition].keys()\n                for repetition in self.split\n            )\n        ):\n            return False\n\n        samples = [\n            (repetition, fold, sample)\n            for repetition in self.split\n            for fold in self.split[repetition]\n            for sample in self.split[repetition][fold]\n        ]\n\n        for repetition, fold, sample in samples:\n            self_train, self_test = self.split[repetition][fold][sample]\n            other_train, other_test = other.split[repetition][fold][sample]\n            if not (np.all(self_train == other_train) and np.all(self_test == other_test)):\n                return False\n        return True\n\n    @classmethod\n    def _from_arff_file(cls, filename: Path) -&gt; OpenMLSplit:  # noqa: C901, PLR0912\n        repetitions = None\n        name = None\n\n        pkl_filename = filename.with_suffix(\".pkl.py3\")\n\n        if pkl_filename.exists():\n            with pkl_filename.open(\"rb\") as fh:\n                # TODO(eddiebergman): Would be good to figure out what _split is and assert it is\n                _split = pickle.load(fh)  # noqa: S301\n            repetitions = _split[\"repetitions\"]\n            name = _split[\"name\"]\n\n        # Cache miss\n        if repetitions is None:\n            # Faster than liac-arff and sufficient in this situation!\n            if not filename.exists():\n                raise FileNotFoundError(f\"Split arff {filename} does not exist!\")\n\n            file_data = arff.load(filename.open(\"r\"), return_type=arff.DENSE_GEN)\n            splits = file_data[\"data\"]\n            name = file_data[\"relation\"]\n            attrnames = [attr[0] for attr in file_data[\"attributes\"]]\n\n            repetitions = OrderedDict()\n\n            type_idx = attrnames.index(\"type\")\n            rowid_idx = attrnames.index(\"rowid\")\n            repeat_idx = attrnames.index(\"repeat\")\n            fold_idx = attrnames.index(\"fold\")\n            sample_idx = attrnames.index(\"sample\") if \"sample\" in attrnames else None\n\n            for line in splits:\n                # A line looks like type, rowid, repeat, fold\n                repetition = int(line[repeat_idx])\n                fold = int(line[fold_idx])\n                sample = 0\n                if sample_idx is not None:\n                    sample = int(line[sample_idx])\n\n                if repetition not in repetitions:\n                    repetitions[repetition] = OrderedDict()\n                if fold not in repetitions[repetition]:\n                    repetitions[repetition][fold] = OrderedDict()\n                if sample not in repetitions[repetition][fold]:\n                    repetitions[repetition][fold][sample] = ([], [])\n                split = repetitions[repetition][fold][sample]\n\n                type_ = line[type_idx]\n                if type_ == \"TRAIN\":\n                    split[0].append(line[rowid_idx])\n                elif type_ == \"TEST\":\n                    split[1].append(line[rowid_idx])\n                else:\n                    raise ValueError(type_)\n\n            for repetition in repetitions:\n                for fold in repetitions[repetition]:\n                    for sample in repetitions[repetition][fold]:\n                        repetitions[repetition][fold][sample] = Split(\n                            np.array(repetitions[repetition][fold][sample][0], dtype=np.int32),\n                            np.array(repetitions[repetition][fold][sample][1], dtype=np.int32),\n                        )\n\n            with pkl_filename.open(\"wb\") as fh:\n                pickle.dump({\"name\": name, \"repetitions\": repetitions}, fh, protocol=2)\n\n        assert name is not None\n        return cls(name, \"\", repetitions)\n\n    def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n        Parameters\n        ----------\n        repeat : int\n            Index of the repeat to retrieve.\n        fold : int\n            Index of the fold to retrieve.\n        sample : int\n            Index of the sample to retrieve.\n\n        Returns\n        -------\n        numpy.ndarray\n            The data split for the specified repeat, fold, and sample.\n\n        Raises\n        ------\n        ValueError\n            If the specified repeat, fold, or sample is not known.\n        \"\"\"\n        if repeat not in self.split:\n            raise ValueError(\"Repeat %s not known\" % str(repeat))\n        if fold not in self.split[repeat]:\n            raise ValueError(\"Fold %s not known\" % str(fold))\n        if sample not in self.split[repeat][fold]:\n            raise ValueError(\"Sample %s not known\" % str(sample))\n        return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSplit.get","title":"<code>get(repeat=0, fold=0, sample=0)</code>","text":"<p>Returns the specified data split from the CrossValidationSplit object.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>Index of the repeat to retrieve.</p> <code>0</code> <code>fold</code> <code>int</code> <p>Index of the fold to retrieve.</p> <code>0</code> <code>sample</code> <code>int</code> <p>Index of the sample to retrieve.</p> <code>0</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>The data split for the specified repeat, fold, and sample.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the specified repeat, fold, or sample is not known.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n    Parameters\n    ----------\n    repeat : int\n        Index of the repeat to retrieve.\n    fold : int\n        Index of the fold to retrieve.\n    sample : int\n        Index of the sample to retrieve.\n\n    Returns\n    -------\n    numpy.ndarray\n        The data split for the specified repeat, fold, and sample.\n\n    Raises\n    ------\n    ValueError\n        If the specified repeat, fold, or sample is not known.\n    \"\"\"\n    if repeat not in self.split:\n        raise ValueError(\"Repeat %s not known\" % str(repeat))\n    if fold not in self.split[repeat]:\n        raise ValueError(\"Fold %s not known\" % str(fold))\n    if sample not in self.split[repeat][fold]:\n        raise ValueError(\"Sample %s not known\" % str(sample))\n    return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSupervisedTask","title":"<code>OpenMLSupervisedTask</code>","text":"<p>               Bases: <code>OpenMLTask</code>, <code>ABC</code></p> <p>OpenML Supervised Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature (the class variable).</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure for the task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure for the task.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the task.</p> <code>None</code> <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLSupervisedTask(OpenMLTask, ABC):\n    \"\"\"OpenML Supervised Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the task type.\n    task_type : str\n        Name of the task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the task.\n    target_name : str\n        Name of the target feature (the class variable).\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure for the task.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure for the task.\n    data_splits_url : str, default=None\n        URL of the data splits for the task.\n    task_id: Union[int, None]\n        Refers to the unique identifier of task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | scipy.sparse.spmatrix,\n        np.ndarray | None,\n    ]:\n        ...\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\"]\n    ) -&gt; tuple[\n        pd.DataFrame,\n        pd.Series | pd.DataFrame | None,\n    ]:\n        ...\n\n    # TODO(eddiebergman): Do all OpenMLSupervisedTask have a `y`?\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n        np.ndarray | pd.Series | pd.DataFrame | None,\n    ]:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        dataset = self.get_dataset()\n        if self.task_type_id not in (\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.SUPERVISED_REGRESSION,\n            TaskType.LEARNING_CURVE,\n        ):\n            raise NotImplementedError(self.task_type)\n\n        X, y, _, _ = dataset.get_data(\n            dataset_format=dataset_format,\n            target=self.target_name,\n        )\n        return X, y\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        task_container = super()._to_dict()\n        oml_input = task_container[\"oml:task_inputs\"][\"oml:input\"]  # type: ignore\n        assert isinstance(oml_input, list)\n\n        oml_input.append({\"@name\": \"target_feature\", \"#text\": self.target_name})\n        return task_container\n\n    @property\n    def estimation_parameters(self) -&gt; dict[str, str] | None:\n        \"\"\"Return the estimation parameters for the task.\"\"\"\n        warnings.warn(\n            \"The estimation_parameters attribute will be \"\n            \"deprecated in the future, please use \"\n            \"estimation_procedure['parameters'] instead\",\n            PendingDeprecationWarning,\n            stacklevel=2,\n        )\n        return self.estimation_procedure[\"parameters\"]\n\n    @estimation_parameters.setter\n    def estimation_parameters(self, est_parameters: dict[str, str] | None) -&gt; None:\n        self.estimation_procedure[\"parameters\"] = est_parameters\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLSupervisedTask.estimation_parameters","title":"<code>estimation_parameters: dict[str, str] | None</code>  <code>property</code> <code>writable</code>","text":"<p>Return the estimation parameters for the task.</p>"},{"location":"reference/tasks/#openml.tasks.OpenMLSupervisedTask.get_X_and_y","title":"<code>get_X_and_y(dataset_format='array')</code>","text":"<pre><code>get_X_and_y(dataset_format: Literal['array'] = 'array') -&gt; tuple[np.ndarray | scipy.sparse.spmatrix, np.ndarray | None]\n</code></pre><pre><code>get_X_and_y(dataset_format: Literal['dataframe']) -&gt; tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X_and_y(\n    self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n    np.ndarray | pd.Series | pd.DataFrame | None,\n]:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    dataset = self.get_dataset()\n    if self.task_type_id not in (\n        TaskType.SUPERVISED_CLASSIFICATION,\n        TaskType.SUPERVISED_REGRESSION,\n        TaskType.LEARNING_CURVE,\n    ):\n        raise NotImplementedError(self.task_type)\n\n    X, y, _, _ = dataset.get_data(\n        dataset_format=dataset_format,\n        target=self.target_name,\n    )\n    return X, y\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask","title":"<code>OpenMLTask</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Task object.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of OpenML task.</p> required <code>task_type_id</code> <code>TaskType</code> <p>Refers to the type of OpenML task.</p> required <code>task_type</code> <code>str</code> <p>Refers to the OpenML task.</p> required <code>data_set_id</code> <code>int</code> <p>Refers to the data.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>Refers to the type of estimates used.</p> <code>1</code> <code>estimation_procedure_type</code> <code>str | None</code> <p>Refers to the type of estimation procedure used for the OpenML task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict[str, str] | None</code> <p>Estimation parameters used for the OpenML task.</p> <code>None</code> <code>evaluation_measure</code> <code>str | None</code> <p>Refers to the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str | None</code> <p>Refers to the URL of the data splits used for the OpenML task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLTask(OpenMLBase):\n    \"\"\"OpenML Task object.\n\n    Parameters\n    ----------\n    task_id: Union[int, None]\n        Refers to the unique identifier of OpenML task.\n    task_type_id: TaskType\n        Refers to the type of OpenML task.\n    task_type: str\n        Refers to the OpenML task.\n    data_set_id: int\n        Refers to the data.\n    estimation_procedure_id: int\n        Refers to the type of estimates used.\n    estimation_procedure_type: str, default=None\n        Refers to the type of estimation procedure used for the OpenML task.\n    estimation_parameters: [Dict[str, str]], default=None\n        Estimation parameters used for the OpenML task.\n    evaluation_measure: str, default=None\n        Refers to the evaluation measure.\n    data_splits_url: str, default=None\n        Refers to the URL of the data splits used for the OpenML task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int | None,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n    ):\n        self.task_id = int(task_id) if task_id is not None else None\n        self.task_type_id = task_type_id\n        self.task_type = task_type\n        self.dataset_id = int(data_set_id)\n        self.evaluation_measure = evaluation_measure\n        self.estimation_procedure: _EstimationProcedure = {\n            \"type\": estimation_procedure_type,\n            \"parameters\": estimation_parameters,\n            \"data_splits_url\": data_splits_url,\n        }\n        self.estimation_procedure_id = estimation_procedure_id\n        self.split: OpenMLSplit | None = None\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"t\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the OpenML ID of this task.\"\"\"\n        return self.task_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        base_server_url = openml.config.get_server_base_url()\n        fields: dict[str, Any] = {\n            \"Task Type Description\": f\"{base_server_url}/tt/{self.task_type_id}\"\n        }\n        if self.task_id is not None:\n            fields[\"Task ID\"] = self.task_id\n            fields[\"Task URL\"] = self.openml_url\n        if self.evaluation_measure is not None:\n            fields[\"Evaluation Measure\"] = self.evaluation_measure\n        if self.estimation_procedure is not None:\n            fields[\"Estimation Procedure\"] = self.estimation_procedure[\"type\"]\n\n        # TODO(eddiebergman): Subclasses could advertise/provide this, instead of having to\n        # have the base class know about it's subclasses.\n        target_name = getattr(self, \"target_name\", None)\n        if target_name is not None:\n            fields[\"Target Feature\"] = target_name\n\n            class_labels = getattr(self, \"class_labels\", None)\n            if class_labels is not None:\n                fields[\"# of Classes\"] = len(class_labels)\n\n            if hasattr(self, \"cost_matrix\"):\n                fields[\"Cost Matrix\"] = \"Available\"\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Task Type Description\",\n            \"Task ID\",\n            \"Task URL\",\n            \"Estimation Procedure\",\n            \"Evaluation Measure\",\n            \"Target Feature\",\n            \"# of Classes\",\n            \"Cost Matrix\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def get_dataset(self) -&gt; datasets.OpenMLDataset:\n        \"\"\"Download dataset associated with task.\"\"\"\n        return datasets.get_dataset(self.dataset_id)\n\n    def get_train_test_split_indices(\n        self,\n        fold: int = 0,\n        repeat: int = 0,\n        sample: int = 0,\n    ) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n        # Replace with retrieve from cache\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.get(repeat=repeat, fold=fold, sample=sample)\n\n    def _download_split(self, cache_file: Path) -&gt; None:\n        # TODO(eddiebergman): Not sure about this try to read and error approach\n        try:\n            with cache_file.open(encoding=\"utf8\"):\n                pass\n        except OSError:\n            split_url = self.estimation_procedure[\"data_splits_url\"]\n            openml._api_calls._download_text_file(\n                source=str(split_url),\n                output_path=str(cache_file),\n            )\n\n    def download_split(self) -&gt; OpenMLSplit:\n        \"\"\"Download the OpenML split for a given task.\"\"\"\n        # TODO(eddiebergman): Can this every be `None`?\n        assert self.task_id is not None\n        cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n        cached_split_file = cache_dir / \"datasplits.arff\"\n\n        try:\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n        except OSError:\n            # Next, download and cache the associated split file\n            self._download_split(cached_split_file)\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n\n        return split\n\n    def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n        \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.repeats, self.split.folds, self.split.samples\n\n    # TODO(eddiebergman): Really need some better typing on all this\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        \"\"\"Creates a dictionary representation of self in a string format (for XML parsing).\"\"\"\n        oml_input = [\n            {\"@name\": \"source_data\", \"#text\": str(self.dataset_id)},\n            {\"@name\": \"estimation_procedure\", \"#text\": str(self.estimation_procedure_id)},\n        ]\n        if self.evaluation_measure is not None:  #\n            oml_input.append({\"@name\": \"evaluation_measures\", \"#text\": self.evaluation_measure})\n\n        return {\n            \"oml:task_inputs\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                \"oml:task_type_id\": self.task_type_id.value,  # This is an int from the enum?\n                \"oml:input\": oml_input,\n            }\n        }\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.task_id = int(xml_response[\"oml:upload_task\"][\"oml:id\"])\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the OpenML ID of this task.</p>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.download_split","title":"<code>download_split()</code>","text":"<p>Download the OpenML split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def download_split(self) -&gt; OpenMLSplit:\n    \"\"\"Download the OpenML split for a given task.\"\"\"\n    # TODO(eddiebergman): Can this every be `None`?\n    assert self.task_id is not None\n    cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n    cached_split_file = cache_dir / \"datasplits.arff\"\n\n    try:\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n    except OSError:\n        # Next, download and cache the associated split file\n        self._download_split(cached_split_file)\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n\n    return split\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.get_dataset","title":"<code>get_dataset()</code>","text":"<p>Download dataset associated with task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_dataset(self) -&gt; datasets.OpenMLDataset:\n    \"\"\"Download dataset associated with task.\"\"\"\n    return datasets.get_dataset(self.dataset_id)\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.get_split_dimensions","title":"<code>get_split_dimensions()</code>","text":"<p>Get the (repeats, folds, samples) of the split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n    \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.repeats, self.split.folds, self.split.samples\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.OpenMLTask.get_train_test_split_indices","title":"<code>get_train_test_split_indices(fold=0, repeat=0, sample=0)</code>","text":"<p>Get the indices of the train and test splits for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_train_test_split_indices(\n    self,\n    fold: int = 0,\n    repeat: int = 0,\n    sample: int = 0,\n) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n    # Replace with retrieve from cache\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.get(repeat=repeat, fold=fold, sample=sample)\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.TaskType","title":"<code>TaskType</code>","text":"<p>               Bases: <code>Enum</code></p> <p>Possible task types as defined in OpenML.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>class TaskType(Enum):\n    \"\"\"Possible task types as defined in OpenML.\"\"\"\n\n    SUPERVISED_CLASSIFICATION = 1\n    SUPERVISED_REGRESSION = 2\n    LEARNING_CURVE = 3\n    SUPERVISED_DATASTREAM_CLASSIFICATION = 4\n    CLUSTERING = 5\n    MACHINE_LEARNING_CHALLENGE = 6\n    SURVIVAL_ANALYSIS = 7\n    SUBGROUP_DISCOVERY = 8\n    MULTITASK_REGRESSION = 9\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.create_task","title":"<code>create_task(task_type, dataset_id, estimation_procedure_id, target_name=None, evaluation_measure=None, **kwargs)</code>","text":"<p>Create a task based on different given attributes.</p> <p>Builds a task object with the function arguments as attributes. The type of the task object built is determined from the task type id. More information on how the arguments (task attributes), relate to the different possible tasks can be found in the individual task objects at the openml.tasks.task module.</p> <p>Parameters:</p> Name Type Description Default <code>task_type</code> <code>TaskType</code> <p>Id of the task type.</p> required <code>dataset_id</code> <code>int</code> <p>The id of the dataset for the task.</p> required <code>target_name</code> <code>str</code> <p>The name of the feature used as a target. At the moment, only optional for the clustering tasks.</p> <code>None</code> <code>estimation_procedure_id</code> <code>int</code> <p>The id of the estimation procedure.</p> required <code>evaluation_measure</code> <code>str</code> <p>The name of the evaluation measure.</p> <code>None</code> <code>kwargs</code> <code>dict</code> <p>Other task attributes that are not mandatory for task upload.</p> <code>{}</code> <p>Returns:</p> Type Description <code>(OpenMLClassificationTask, OpenMLRegressionTask)</code> <code>(OpenMLLearningCurveTask, OpenMLClusteringTask)</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def create_task(\n    task_type: TaskType,\n    dataset_id: int,\n    estimation_procedure_id: int,\n    target_name: str | None = None,\n    evaluation_measure: str | None = None,\n    **kwargs: Any,\n) -&gt; (\n    OpenMLClassificationTask | OpenMLRegressionTask | OpenMLLearningCurveTask | OpenMLClusteringTask\n):\n    \"\"\"Create a task based on different given attributes.\n\n    Builds a task object with the function arguments as\n    attributes. The type of the task object built is\n    determined from the task type id.\n    More information on how the arguments (task attributes),\n    relate to the different possible tasks can be found in\n    the individual task objects at the openml.tasks.task\n    module.\n\n    Parameters\n    ----------\n    task_type : TaskType\n        Id of the task type.\n    dataset_id : int\n        The id of the dataset for the task.\n    target_name : str, optional\n        The name of the feature used as a target.\n        At the moment, only optional for the clustering tasks.\n    estimation_procedure_id : int\n        The id of the estimation procedure.\n    evaluation_measure : str, optional\n        The name of the evaluation measure.\n    kwargs : dict, optional\n        Other task attributes that are not mandatory\n        for task upload.\n\n    Returns\n    -------\n    OpenMLClassificationTask, OpenMLRegressionTask,\n    OpenMLLearningCurveTask, OpenMLClusteringTask\n    \"\"\"\n    if task_type == TaskType.CLUSTERING:\n        task_cls = OpenMLClusteringTask\n    elif task_type == TaskType.LEARNING_CURVE:\n        task_cls = OpenMLLearningCurveTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_CLASSIFICATION:\n        task_cls = OpenMLClassificationTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_REGRESSION:\n        task_cls = OpenMLRegressionTask  # type: ignore\n    else:\n        raise NotImplementedError(f\"Task type {task_type:d} not supported.\")\n\n    return task_cls(\n        task_type_id=task_type,\n        task_type=\"None\",  # TODO: refactor to get task type string from ID.\n        data_set_id=dataset_id,\n        target_name=target_name,\n        estimation_procedure_id=estimation_procedure_id,\n        evaluation_measure=evaluation_measure,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.delete_task","title":"<code>delete_task(task_id)</code>","text":"<p>Delete task with id <code>task_id</code> from the OpenML server.</p> <p>You can only delete tasks which you created and have no runs associated with them.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>OpenML id of the task</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def delete_task(task_id: int) -&gt; bool:\n    \"\"\"Delete task with id `task_id` from the OpenML server.\n\n    You can only delete tasks which you created and have\n    no runs associated with them.\n\n    Parameters\n    ----------\n    task_id : int\n        OpenML id of the task\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"task\", task_id)\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.get_task","title":"<code>get_task(task_id, *dataset_args, download_splits=None, **get_dataset_kwargs)</code>","text":"<p>Download OpenML task for a given task ID.</p> <p>Downloads the task representation. By default, this will also download the data splits and the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by default.</p> <p>Use the <code>download_splits</code> parameter to control whether the splits are downloaded. Moreover, you may pass additional parameter (args or kwargs) that are passed to :meth:<code>openml.datasets.get_dataset</code>. For backwards compatibility, if <code>download_data</code> is passed as an additional parameter and <code>download_splits</code> is not explicitly set, <code>download_data</code> also overrules <code>download_splits</code>'s value (deprecated from Version 0.15.0 onwards).</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The OpenML task id of the task to download.</p> required <code>download_splits</code> <code>bool | None</code> <p>Whether to download the splits as well. From version 0.15.0 onwards this is independent of download_data and will default to <code>False</code>.</p> <code>None</code> <code>dataset_args</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <code>get_dataset_kwargs</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <p>Returns:</p> Name Type Description <code>task</code> <code>OpenMLTask</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_task(\n    task_id: int,\n    *dataset_args: Any,\n    download_splits: bool | None = None,\n    **get_dataset_kwargs: Any,\n) -&gt; OpenMLTask:\n    \"\"\"Download OpenML task for a given task ID.\n\n    Downloads the task representation. By default, this will also download the data splits and\n    the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by\n    default.\n\n    Use the `download_splits` parameter to control whether the splits are downloaded.\n    Moreover, you may pass additional parameter (args or kwargs) that are passed to\n    :meth:`openml.datasets.get_dataset`.\n    For backwards compatibility, if `download_data` is passed as an additional parameter and\n    `download_splits` is not explicitly set, `download_data` also overrules `download_splits`'s\n    value (deprecated from Version 0.15.0 onwards).\n\n    Parameters\n    ----------\n    task_id : int\n        The OpenML task id of the task to download.\n    download_splits: bool (default=True)\n        Whether to download the splits as well. From version 0.15.0 onwards this is independent\n        of download_data and will default to ``False``.\n    dataset_args, get_dataset_kwargs :\n        Args and kwargs can be used pass optional parameters to :meth:`openml.datasets.get_dataset`.\n        This includes `download_data`. If set to True the splits are downloaded as well\n        (deprecated from Version 0.15.0 onwards). The args are only present for backwards\n        compatibility and will be removed from version 0.15.0 onwards.\n\n    Returns\n    -------\n    task: OpenMLTask\n    \"\"\"\n    if download_splits is None:\n        # TODO(0.15): Switch download splits to False by default, adjust typing above, adjust\n        #  documentation above, and remove warning.\n        warnings.warn(\n            \"Starting from Version 0.15.0 `download_splits` will default to ``False`` instead \"\n            \"of ``True`` and be independent from `download_data`. To disable this message until \"\n            \"version 0.15 explicitly set `download_splits` to a bool.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n        download_splits = get_dataset_kwargs.get(\"download_data\", True)\n\n    if not isinstance(task_id, int):\n        # TODO(0.15): Remove warning\n        warnings.warn(\n            \"Task id must be specified as `int` from 0.14.0 onwards.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n\n    try:\n        task_id = int(task_id)\n    except (ValueError, TypeError) as e:\n        raise ValueError(\"Dataset ID is neither an Integer nor can be cast to an Integer.\") from e\n\n    tid_cache_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id)\n\n    try:\n        task = _get_task_description(task_id)\n        dataset = get_dataset(task.dataset_id, *dataset_args, **get_dataset_kwargs)\n        # List of class labels available in dataset description\n        # Including class labels as part of task meta data handles\n        #   the case where data download was initially disabled\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            task.class_labels = dataset.retrieve_class_labels(task.target_name)\n        # Clustering tasks do not have class labels\n        # and do not offer download_split\n        if download_splits and isinstance(task, OpenMLSupervisedTask):\n            task.download_split()\n    except Exception as e:\n        openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir)\n        raise e\n\n    return task\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.get_tasks","title":"<code>get_tasks(task_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download tasks.</p> <p>This function iterates :meth:<code>openml.tasks.get_task</code>.</p> <p>Parameters:</p> Name Type Description Default <code>task_ids</code> <code>List[int]</code> <p>A list of task ids to download.</p> required <code>download_data</code> <code>bool(default=True)</code> <p>Option to trigger download of data along with the meta data.</p> <code>True</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description.</p> <code>True</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def get_tasks(\n    task_ids: list[int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLTask]:\n    \"\"\"Download tasks.\n\n    This function iterates :meth:`openml.tasks.get_task`.\n\n    Parameters\n    ----------\n    task_ids : List[int]\n        A list of task ids to download.\n    download_data : bool (default = True)\n        Option to trigger download of data along with the meta data.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    tasks = []\n    for task_id in task_ids:\n        tasks.append(get_task(task_id, download_data, download_qualities))\n    return tasks\n</code></pre>"},{"location":"reference/tasks/#openml.tasks.list_tasks","title":"<code>list_tasks(task_type=None, offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<p>Return a number of tasks having the given tag and task_type</p> <p>Parameters:</p> Name Type Description Default <code>Filter</code> required <code>it</code> required <code>type</code> required <code>task_type</code> <code>TaskType</code> <p>Refers to the type of task.</p> <code>None</code> <code>offset</code> <code>int</code> <p>the number of tasks to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of tasks to show</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: data_tag, status, data_id, data_name, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a dictionary containing the following information: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> <code>dataframe</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a row in the data frame containing the following information as columns: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def list_tasks(\n    task_type: TaskType | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a number of tasks having the given tag and task_type\n\n    Parameters\n    ----------\n    Filter task_type is separated from the other filters because\n    it is used as task_type in the task description, but it is named\n    type when used as a filter in list tasks call.\n    task_type : TaskType, optional\n        Refers to the type of task.\n    offset : int, optional\n        the number of tasks to skip, starting from the first\n    size : int, optional\n        the maximum number of tasks to show\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: data_tag, status, data_id, data_name,\n        number_instances, number_features,\n        number_classes, number_missing_values.\n\n    Returns\n    -------\n    dict\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a dictionary containing the following information:\n        task id, dataset id, task_type and status. If qualities are calculated\n        for the associated dataset, some of these are also returned.\n    dataframe\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a row in the data frame containing the following information\n        as columns: task id, dataset id, task_type and status. If qualities are\n        calculated for the associated dataset, some of these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_tasks,\n        task_type=task_type,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/functions/","title":"functions","text":""},{"location":"reference/tasks/functions/#openml.tasks.functions.__list_tasks","title":"<code>__list_tasks(api_call, output_format='dict')</code>","text":"<p>Returns a dictionary or a Pandas DataFrame with information about OpenML tasks.</p> <p>Parameters:</p> Name Type Description Default <code>api_call</code> <code>str</code> <p>The API call specifying which tasks to return.</p> required <code>output_format</code> <code>str in {'dict', 'dataframe'}</code> <p>Output format for the returned object.</p> <code>'dict'</code> <p>Returns:</p> Type Description <code>Union[Dict, DataFrame]</code> <p>A dictionary or a Pandas DataFrame with information about OpenML tasks.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the XML returned by the OpenML API does not contain 'oml:tasks', '@xmlns:oml', or has an incorrect value for '@xmlns:oml'.</p> <code>KeyError</code> <p>If an invalid key is found in the XML for a task.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def __list_tasks(  # noqa: PLR0912, C901\n    api_call: str,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n) -&gt; dict | pd.DataFrame:\n    \"\"\"Returns a dictionary or a Pandas DataFrame with information about OpenML tasks.\n\n    Parameters\n    ----------\n    api_call : str\n        The API call specifying which tasks to return.\n    output_format : str in {\"dict\", \"dataframe\"}\n        Output format for the returned object.\n\n    Returns\n    -------\n    Union[Dict, pd.DataFrame]\n        A dictionary or a Pandas DataFrame with information about OpenML tasks.\n\n    Raises\n    ------\n    ValueError\n        If the XML returned by the OpenML API does not contain 'oml:tasks', '@xmlns:oml',\n        or has an incorrect value for '@xmlns:oml'.\n    KeyError\n        If an invalid key is found in the XML for a task.\n    \"\"\"\n    xml_string = openml._api_calls._perform_api_call(api_call, \"get\")\n    tasks_dict = xmltodict.parse(xml_string, force_list=(\"oml:task\", \"oml:input\"))\n    # Minimalistic check if the XML is useful\n    if \"oml:tasks\" not in tasks_dict:\n        raise ValueError(f'Error in return XML, does not contain \"oml:runs\": {tasks_dict}')\n\n    if \"@xmlns:oml\" not in tasks_dict[\"oml:tasks\"]:\n        raise ValueError(\n            f'Error in return XML, does not contain \"oml:runs\"/@xmlns:oml: {tasks_dict}'\n        )\n\n    if tasks_dict[\"oml:tasks\"][\"@xmlns:oml\"] != \"http://openml.org/openml\":\n        raise ValueError(\n            \"Error in return XML, value of  \"\n            '\"oml:runs\"/@xmlns:oml is not '\n            '\"http://openml.org/openml\": %s' % str(tasks_dict),\n        )\n\n    assert isinstance(tasks_dict[\"oml:tasks\"][\"oml:task\"], list), type(tasks_dict[\"oml:tasks\"])\n\n    tasks = {}\n    procs = _get_estimation_procedure_list()\n    proc_dict = {x[\"id\"]: x for x in procs}\n\n    for task_ in tasks_dict[\"oml:tasks\"][\"oml:task\"]:\n        tid = None\n        try:\n            tid = int(task_[\"oml:task_id\"])\n            task_type_int = int(task_[\"oml:task_type_id\"])\n            try:\n                task_type_id = TaskType(task_type_int)\n            except ValueError as e:\n                warnings.warn(\n                    f\"Could not create task type id for {task_type_int} due to error {e}\",\n                    RuntimeWarning,\n                    stacklevel=2,\n                )\n                continue\n\n            task = {\n                \"tid\": tid,\n                \"ttid\": task_type_id,\n                \"did\": int(task_[\"oml:did\"]),\n                \"name\": task_[\"oml:name\"],\n                \"task_type\": task_[\"oml:task_type\"],\n                \"status\": task_[\"oml:status\"],\n            }\n\n            # Other task inputs\n            for _input in task_.get(\"oml:input\", []):\n                if _input[\"@name\"] == \"estimation_procedure\":\n                    task[_input[\"@name\"]] = proc_dict[int(_input[\"#text\"])][\"name\"]\n                else:\n                    value = _input.get(\"#text\")\n                    task[_input[\"@name\"]] = value\n\n            # The number of qualities can range from 0 to infinity\n            for quality in task_.get(\"oml:quality\", []):\n                if \"#text\" not in quality:\n                    quality_value = 0.0\n                else:\n                    quality[\"#text\"] = float(quality[\"#text\"])\n                    if abs(int(quality[\"#text\"]) - quality[\"#text\"]) &lt; 0.0000001:\n                        quality[\"#text\"] = int(quality[\"#text\"])\n                    quality_value = quality[\"#text\"]\n                task[quality[\"@name\"]] = quality_value\n            tasks[tid] = task\n        except KeyError as e:\n            if tid is not None:\n                warnings.warn(\n                    \"Invalid xml for task %d: %s\\nFrom %s\" % (tid, e, task_),\n                    RuntimeWarning,\n                    stacklevel=2,\n                )\n            else:\n                warnings.warn(f\"Could not find key {e} in {task_}!\", RuntimeWarning, stacklevel=2)\n\n    if output_format == \"dataframe\":\n        tasks = pd.DataFrame.from_dict(tasks, orient=\"index\")\n\n    return tasks\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.create_task","title":"<code>create_task(task_type, dataset_id, estimation_procedure_id, target_name=None, evaluation_measure=None, **kwargs)</code>","text":"<p>Create a task based on different given attributes.</p> <p>Builds a task object with the function arguments as attributes. The type of the task object built is determined from the task type id. More information on how the arguments (task attributes), relate to the different possible tasks can be found in the individual task objects at the openml.tasks.task module.</p> <p>Parameters:</p> Name Type Description Default <code>task_type</code> <code>TaskType</code> <p>Id of the task type.</p> required <code>dataset_id</code> <code>int</code> <p>The id of the dataset for the task.</p> required <code>target_name</code> <code>str</code> <p>The name of the feature used as a target. At the moment, only optional for the clustering tasks.</p> <code>None</code> <code>estimation_procedure_id</code> <code>int</code> <p>The id of the estimation procedure.</p> required <code>evaluation_measure</code> <code>str</code> <p>The name of the evaluation measure.</p> <code>None</code> <code>kwargs</code> <code>dict</code> <p>Other task attributes that are not mandatory for task upload.</p> <code>{}</code> <p>Returns:</p> Type Description <code>(OpenMLClassificationTask, OpenMLRegressionTask)</code> <code>(OpenMLLearningCurveTask, OpenMLClusteringTask)</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def create_task(\n    task_type: TaskType,\n    dataset_id: int,\n    estimation_procedure_id: int,\n    target_name: str | None = None,\n    evaluation_measure: str | None = None,\n    **kwargs: Any,\n) -&gt; (\n    OpenMLClassificationTask | OpenMLRegressionTask | OpenMLLearningCurveTask | OpenMLClusteringTask\n):\n    \"\"\"Create a task based on different given attributes.\n\n    Builds a task object with the function arguments as\n    attributes. The type of the task object built is\n    determined from the task type id.\n    More information on how the arguments (task attributes),\n    relate to the different possible tasks can be found in\n    the individual task objects at the openml.tasks.task\n    module.\n\n    Parameters\n    ----------\n    task_type : TaskType\n        Id of the task type.\n    dataset_id : int\n        The id of the dataset for the task.\n    target_name : str, optional\n        The name of the feature used as a target.\n        At the moment, only optional for the clustering tasks.\n    estimation_procedure_id : int\n        The id of the estimation procedure.\n    evaluation_measure : str, optional\n        The name of the evaluation measure.\n    kwargs : dict, optional\n        Other task attributes that are not mandatory\n        for task upload.\n\n    Returns\n    -------\n    OpenMLClassificationTask, OpenMLRegressionTask,\n    OpenMLLearningCurveTask, OpenMLClusteringTask\n    \"\"\"\n    if task_type == TaskType.CLUSTERING:\n        task_cls = OpenMLClusteringTask\n    elif task_type == TaskType.LEARNING_CURVE:\n        task_cls = OpenMLLearningCurveTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_CLASSIFICATION:\n        task_cls = OpenMLClassificationTask  # type: ignore\n    elif task_type == TaskType.SUPERVISED_REGRESSION:\n        task_cls = OpenMLRegressionTask  # type: ignore\n    else:\n        raise NotImplementedError(f\"Task type {task_type:d} not supported.\")\n\n    return task_cls(\n        task_type_id=task_type,\n        task_type=\"None\",  # TODO: refactor to get task type string from ID.\n        data_set_id=dataset_id,\n        target_name=target_name,\n        estimation_procedure_id=estimation_procedure_id,\n        evaluation_measure=evaluation_measure,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.delete_task","title":"<code>delete_task(task_id)</code>","text":"<p>Delete task with id <code>task_id</code> from the OpenML server.</p> <p>You can only delete tasks which you created and have no runs associated with them.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>OpenML id of the task</p> required <p>Returns:</p> Type Description <code>bool</code> <p>True if the deletion was successful. False otherwise.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def delete_task(task_id: int) -&gt; bool:\n    \"\"\"Delete task with id `task_id` from the OpenML server.\n\n    You can only delete tasks which you created and have\n    no runs associated with them.\n\n    Parameters\n    ----------\n    task_id : int\n        OpenML id of the task\n\n    Returns\n    -------\n    bool\n        True if the deletion was successful. False otherwise.\n    \"\"\"\n    return openml.utils._delete_entity(\"task\", task_id)\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.get_task","title":"<code>get_task(task_id, *dataset_args, download_splits=None, **get_dataset_kwargs)</code>","text":"<p>Download OpenML task for a given task ID.</p> <p>Downloads the task representation. By default, this will also download the data splits and the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by default.</p> <p>Use the <code>download_splits</code> parameter to control whether the splits are downloaded. Moreover, you may pass additional parameter (args or kwargs) that are passed to :meth:<code>openml.datasets.get_dataset</code>. For backwards compatibility, if <code>download_data</code> is passed as an additional parameter and <code>download_splits</code> is not explicitly set, <code>download_data</code> also overrules <code>download_splits</code>'s value (deprecated from Version 0.15.0 onwards).</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int</code> <p>The OpenML task id of the task to download.</p> required <code>download_splits</code> <code>bool | None</code> <p>Whether to download the splits as well. From version 0.15.0 onwards this is independent of download_data and will default to <code>False</code>.</p> <code>None</code> <code>dataset_args</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <code>get_dataset_kwargs</code> <code>Any</code> <p>Args and kwargs can be used pass optional parameters to :meth:<code>openml.datasets.get_dataset</code>. This includes <code>download_data</code>. If set to True the splits are downloaded as well (deprecated from Version 0.15.0 onwards). The args are only present for backwards compatibility and will be removed from version 0.15.0 onwards.</p> <code>()</code> <p>Returns:</p> Name Type Description <code>task</code> <code>OpenMLTask</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>@openml.utils.thread_safe_if_oslo_installed\ndef get_task(\n    task_id: int,\n    *dataset_args: Any,\n    download_splits: bool | None = None,\n    **get_dataset_kwargs: Any,\n) -&gt; OpenMLTask:\n    \"\"\"Download OpenML task for a given task ID.\n\n    Downloads the task representation. By default, this will also download the data splits and\n    the dataset. From version 0.15.0 onwards, the splits nor the dataset will not be downloaded by\n    default.\n\n    Use the `download_splits` parameter to control whether the splits are downloaded.\n    Moreover, you may pass additional parameter (args or kwargs) that are passed to\n    :meth:`openml.datasets.get_dataset`.\n    For backwards compatibility, if `download_data` is passed as an additional parameter and\n    `download_splits` is not explicitly set, `download_data` also overrules `download_splits`'s\n    value (deprecated from Version 0.15.0 onwards).\n\n    Parameters\n    ----------\n    task_id : int\n        The OpenML task id of the task to download.\n    download_splits: bool (default=True)\n        Whether to download the splits as well. From version 0.15.0 onwards this is independent\n        of download_data and will default to ``False``.\n    dataset_args, get_dataset_kwargs :\n        Args and kwargs can be used pass optional parameters to :meth:`openml.datasets.get_dataset`.\n        This includes `download_data`. If set to True the splits are downloaded as well\n        (deprecated from Version 0.15.0 onwards). The args are only present for backwards\n        compatibility and will be removed from version 0.15.0 onwards.\n\n    Returns\n    -------\n    task: OpenMLTask\n    \"\"\"\n    if download_splits is None:\n        # TODO(0.15): Switch download splits to False by default, adjust typing above, adjust\n        #  documentation above, and remove warning.\n        warnings.warn(\n            \"Starting from Version 0.15.0 `download_splits` will default to ``False`` instead \"\n            \"of ``True`` and be independent from `download_data`. To disable this message until \"\n            \"version 0.15 explicitly set `download_splits` to a bool.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n        download_splits = get_dataset_kwargs.get(\"download_data\", True)\n\n    if not isinstance(task_id, int):\n        # TODO(0.15): Remove warning\n        warnings.warn(\n            \"Task id must be specified as `int` from 0.14.0 onwards.\",\n            FutureWarning,\n            stacklevel=3,\n        )\n\n    try:\n        task_id = int(task_id)\n    except (ValueError, TypeError) as e:\n        raise ValueError(\"Dataset ID is neither an Integer nor can be cast to an Integer.\") from e\n\n    tid_cache_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id)\n\n    try:\n        task = _get_task_description(task_id)\n        dataset = get_dataset(task.dataset_id, *dataset_args, **get_dataset_kwargs)\n        # List of class labels available in dataset description\n        # Including class labels as part of task meta data handles\n        #   the case where data download was initially disabled\n        if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):\n            task.class_labels = dataset.retrieve_class_labels(task.target_name)\n        # Clustering tasks do not have class labels\n        # and do not offer download_split\n        if download_splits and isinstance(task, OpenMLSupervisedTask):\n            task.download_split()\n    except Exception as e:\n        openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir)\n        raise e\n\n    return task\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.get_tasks","title":"<code>get_tasks(task_ids, download_data=True, download_qualities=True)</code>","text":"<p>Download tasks.</p> <p>This function iterates :meth:<code>openml.tasks.get_task</code>.</p> <p>Parameters:</p> Name Type Description Default <code>task_ids</code> <code>List[int]</code> <p>A list of task ids to download.</p> required <code>download_data</code> <code>bool(default=True)</code> <p>Option to trigger download of data along with the meta data.</p> <code>True</code> <code>download_qualities</code> <code>bool(default=True)</code> <p>Option to download 'qualities' meta-data in addition to the minimal dataset description.</p> <code>True</code> <p>Returns:</p> Type Description <code>list</code> Source code in <code>openml/tasks/functions.py</code> <pre><code>def get_tasks(\n    task_ids: list[int],\n    download_data: bool = True,  # noqa: FBT001, FBT002\n    download_qualities: bool = True,  # noqa: FBT001, FBT002\n) -&gt; list[OpenMLTask]:\n    \"\"\"Download tasks.\n\n    This function iterates :meth:`openml.tasks.get_task`.\n\n    Parameters\n    ----------\n    task_ids : List[int]\n        A list of task ids to download.\n    download_data : bool (default = True)\n        Option to trigger download of data along with the meta data.\n    download_qualities : bool (default=True)\n        Option to download 'qualities' meta-data in addition to the minimal dataset description.\n\n    Returns\n    -------\n    list\n    \"\"\"\n    tasks = []\n    for task_id in task_ids:\n        tasks.append(get_task(task_id, download_data, download_qualities))\n    return tasks\n</code></pre>"},{"location":"reference/tasks/functions/#openml.tasks.functions.list_tasks","title":"<code>list_tasks(task_type=None, offset=None, size=None, tag=None, output_format='dict', **kwargs)</code>","text":"<p>Return a number of tasks having the given tag and task_type</p> <p>Parameters:</p> Name Type Description Default <code>Filter</code> required <code>it</code> required <code>type</code> required <code>task_type</code> <code>TaskType</code> <p>Refers to the type of task.</p> <code>None</code> <code>offset</code> <code>int</code> <p>the number of tasks to skip, starting from the first</p> <code>None</code> <code>size</code> <code>int</code> <p>the maximum number of tasks to show</p> <code>None</code> <code>tag</code> <code>str</code> <p>the tag to include</p> <code>None</code> <code>output_format</code> <code>Literal['dict', 'dataframe']</code> <p>The parameter decides the format of the output. - If 'dict' the output is a dict of dict - If 'dataframe' the output is a pandas DataFrame</p> <code>'dict'</code> <code>kwargs</code> <code>Any</code> <p>Legal filter operators: data_tag, status, data_id, data_name, number_instances, number_features, number_classes, number_missing_values.</p> <code>{}</code> <p>Returns:</p> Type Description <code>dict</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a dictionary containing the following information: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> <code>dataframe</code> <p>All tasks having the given task_type and the give tag. Every task is represented by a row in the data frame containing the following information as columns: task id, dataset id, task_type and status. If qualities are calculated for the associated dataset, some of these are also returned.</p> Source code in <code>openml/tasks/functions.py</code> <pre><code>def list_tasks(\n    task_type: TaskType | None = None,\n    offset: int | None = None,\n    size: int | None = None,\n    tag: str | None = None,\n    output_format: Literal[\"dict\", \"dataframe\"] = \"dict\",\n    **kwargs: Any,\n) -&gt; dict | pd.DataFrame:\n    \"\"\"\n    Return a number of tasks having the given tag and task_type\n\n    Parameters\n    ----------\n    Filter task_type is separated from the other filters because\n    it is used as task_type in the task description, but it is named\n    type when used as a filter in list tasks call.\n    task_type : TaskType, optional\n        Refers to the type of task.\n    offset : int, optional\n        the number of tasks to skip, starting from the first\n    size : int, optional\n        the maximum number of tasks to show\n    tag : str, optional\n        the tag to include\n    output_format: str, optional (default='dict')\n        The parameter decides the format of the output.\n        - If 'dict' the output is a dict of dict\n        - If 'dataframe' the output is a pandas DataFrame\n    kwargs: dict, optional\n        Legal filter operators: data_tag, status, data_id, data_name,\n        number_instances, number_features,\n        number_classes, number_missing_values.\n\n    Returns\n    -------\n    dict\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a dictionary containing the following information:\n        task id, dataset id, task_type and status. If qualities are calculated\n        for the associated dataset, some of these are also returned.\n    dataframe\n        All tasks having the given task_type and the give tag. Every task is\n        represented by a row in the data frame containing the following information\n        as columns: task id, dataset id, task_type and status. If qualities are\n        calculated for the associated dataset, some of these are also returned.\n    \"\"\"\n    if output_format not in [\"dataframe\", \"dict\"]:\n        raise ValueError(\n            \"Invalid output format selected. \" \"Only 'dict' or 'dataframe' applicable.\",\n        )\n    # TODO: [0.15]\n    if output_format == \"dict\":\n        msg = (\n            \"Support for `output_format` of 'dict' will be removed in 0.15 \"\n            \"and pandas dataframes will be returned instead. To ensure your code \"\n            \"will continue to work, use `output_format`='dataframe'.\"\n        )\n        warnings.warn(msg, category=FutureWarning, stacklevel=2)\n    return openml.utils._list_all(  # type: ignore\n        list_output_format=output_format,  # type: ignore\n        listing_call=_list_tasks,\n        task_type=task_type,\n        offset=offset,\n        size=size,\n        tag=tag,\n        **kwargs,\n    )\n</code></pre>"},{"location":"reference/tasks/split/","title":"split","text":""},{"location":"reference/tasks/split/#openml.tasks.split.OpenMLSplit","title":"<code>OpenMLSplit</code>","text":"<p>OpenML Split object.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>int or str</code> required <code>description</code> <code>str</code> required <code>split</code> <code>dict</code> required Source code in <code>openml/tasks/split.py</code> <pre><code>class OpenMLSplit:\n    \"\"\"OpenML Split object.\n\n    Parameters\n    ----------\n    name : int or str\n    description : str\n    split : dict\n    \"\"\"\n\n    def __init__(\n        self,\n        name: int | str,\n        description: str,\n        split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]],\n    ):\n        self.description = description\n        self.name = name\n        self.split: dict[int, dict[int, dict[int, tuple[np.ndarray, np.ndarray]]]] = {}\n\n        # Add splits according to repetition\n        for repetition in split:\n            _rep = int(repetition)\n            self.split[_rep] = OrderedDict()\n            for fold in split[_rep]:\n                self.split[_rep][fold] = OrderedDict()\n                for sample in split[_rep][fold]:\n                    self.split[_rep][fold][sample] = split[_rep][fold][sample]\n\n        self.repeats = len(self.split)\n\n        # TODO(eddiebergman): Better error message\n        if any(len(self.split[0]) != len(self.split[i]) for i in range(self.repeats)):\n            raise ValueError(\"\")\n\n        self.folds = len(self.split[0])\n        self.samples = len(self.split[0][0])\n\n    def __eq__(self, other: Any) -&gt; bool:\n        if (\n            (not isinstance(self, type(other)))\n            or self.name != other.name\n            or self.description != other.description\n            or self.split.keys() != other.split.keys()\n            or any(\n                self.split[repetition].keys() != other.split[repetition].keys()\n                for repetition in self.split\n            )\n        ):\n            return False\n\n        samples = [\n            (repetition, fold, sample)\n            for repetition in self.split\n            for fold in self.split[repetition]\n            for sample in self.split[repetition][fold]\n        ]\n\n        for repetition, fold, sample in samples:\n            self_train, self_test = self.split[repetition][fold][sample]\n            other_train, other_test = other.split[repetition][fold][sample]\n            if not (np.all(self_train == other_train) and np.all(self_test == other_test)):\n                return False\n        return True\n\n    @classmethod\n    def _from_arff_file(cls, filename: Path) -&gt; OpenMLSplit:  # noqa: C901, PLR0912\n        repetitions = None\n        name = None\n\n        pkl_filename = filename.with_suffix(\".pkl.py3\")\n\n        if pkl_filename.exists():\n            with pkl_filename.open(\"rb\") as fh:\n                # TODO(eddiebergman): Would be good to figure out what _split is and assert it is\n                _split = pickle.load(fh)  # noqa: S301\n            repetitions = _split[\"repetitions\"]\n            name = _split[\"name\"]\n\n        # Cache miss\n        if repetitions is None:\n            # Faster than liac-arff and sufficient in this situation!\n            if not filename.exists():\n                raise FileNotFoundError(f\"Split arff {filename} does not exist!\")\n\n            file_data = arff.load(filename.open(\"r\"), return_type=arff.DENSE_GEN)\n            splits = file_data[\"data\"]\n            name = file_data[\"relation\"]\n            attrnames = [attr[0] for attr in file_data[\"attributes\"]]\n\n            repetitions = OrderedDict()\n\n            type_idx = attrnames.index(\"type\")\n            rowid_idx = attrnames.index(\"rowid\")\n            repeat_idx = attrnames.index(\"repeat\")\n            fold_idx = attrnames.index(\"fold\")\n            sample_idx = attrnames.index(\"sample\") if \"sample\" in attrnames else None\n\n            for line in splits:\n                # A line looks like type, rowid, repeat, fold\n                repetition = int(line[repeat_idx])\n                fold = int(line[fold_idx])\n                sample = 0\n                if sample_idx is not None:\n                    sample = int(line[sample_idx])\n\n                if repetition not in repetitions:\n                    repetitions[repetition] = OrderedDict()\n                if fold not in repetitions[repetition]:\n                    repetitions[repetition][fold] = OrderedDict()\n                if sample not in repetitions[repetition][fold]:\n                    repetitions[repetition][fold][sample] = ([], [])\n                split = repetitions[repetition][fold][sample]\n\n                type_ = line[type_idx]\n                if type_ == \"TRAIN\":\n                    split[0].append(line[rowid_idx])\n                elif type_ == \"TEST\":\n                    split[1].append(line[rowid_idx])\n                else:\n                    raise ValueError(type_)\n\n            for repetition in repetitions:\n                for fold in repetitions[repetition]:\n                    for sample in repetitions[repetition][fold]:\n                        repetitions[repetition][fold][sample] = Split(\n                            np.array(repetitions[repetition][fold][sample][0], dtype=np.int32),\n                            np.array(repetitions[repetition][fold][sample][1], dtype=np.int32),\n                        )\n\n            with pkl_filename.open(\"wb\") as fh:\n                pickle.dump({\"name\": name, \"repetitions\": repetitions}, fh, protocol=2)\n\n        assert name is not None\n        return cls(name, \"\", repetitions)\n\n    def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n        Parameters\n        ----------\n        repeat : int\n            Index of the repeat to retrieve.\n        fold : int\n            Index of the fold to retrieve.\n        sample : int\n            Index of the sample to retrieve.\n\n        Returns\n        -------\n        numpy.ndarray\n            The data split for the specified repeat, fold, and sample.\n\n        Raises\n        ------\n        ValueError\n            If the specified repeat, fold, or sample is not known.\n        \"\"\"\n        if repeat not in self.split:\n            raise ValueError(\"Repeat %s not known\" % str(repeat))\n        if fold not in self.split[repeat]:\n            raise ValueError(\"Fold %s not known\" % str(fold))\n        if sample not in self.split[repeat][fold]:\n            raise ValueError(\"Sample %s not known\" % str(sample))\n        return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/split/#openml.tasks.split.OpenMLSplit.get","title":"<code>get(repeat=0, fold=0, sample=0)</code>","text":"<p>Returns the specified data split from the CrossValidationSplit object.</p> <p>Parameters:</p> Name Type Description Default <code>repeat</code> <code>int</code> <p>Index of the repeat to retrieve.</p> <code>0</code> <code>fold</code> <code>int</code> <p>Index of the fold to retrieve.</p> <code>0</code> <code>sample</code> <code>int</code> <p>Index of the sample to retrieve.</p> <code>0</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>The data split for the specified repeat, fold, and sample.</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If the specified repeat, fold, or sample is not known.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>def get(self, repeat: int = 0, fold: int = 0, sample: int = 0) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Returns the specified data split from the CrossValidationSplit object.\n\n    Parameters\n    ----------\n    repeat : int\n        Index of the repeat to retrieve.\n    fold : int\n        Index of the fold to retrieve.\n    sample : int\n        Index of the sample to retrieve.\n\n    Returns\n    -------\n    numpy.ndarray\n        The data split for the specified repeat, fold, and sample.\n\n    Raises\n    ------\n    ValueError\n        If the specified repeat, fold, or sample is not known.\n    \"\"\"\n    if repeat not in self.split:\n        raise ValueError(\"Repeat %s not known\" % str(repeat))\n    if fold not in self.split[repeat]:\n        raise ValueError(\"Fold %s not known\" % str(fold))\n    if sample not in self.split[repeat][fold]:\n        raise ValueError(\"Sample %s not known\" % str(sample))\n    return self.split[repeat][fold][sample]\n</code></pre>"},{"location":"reference/tasks/split/#openml.tasks.split.Split","title":"<code>Split</code>","text":"<p>               Bases: <code>NamedTuple</code></p> <p>A single split of a dataset.</p> Source code in <code>openml/tasks/split.py</code> <pre><code>class Split(NamedTuple):\n    \"\"\"A single split of a dataset.\"\"\"\n\n    train: np.ndarray\n    test: np.ndarray\n</code></pre>"},{"location":"reference/tasks/task/","title":"task","text":""},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLClassificationTask","title":"<code>OpenMLClassificationTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Classification task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the Classification task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the Classification task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target variable.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the Classification task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the Classification task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the Classification task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Classification task (if it already exists on OpenML).</p> <code>None</code> <code>class_labels</code> <code>List of str</code> <p>A list of class labels (for classification tasks).</p> <code>None</code> <code>cost_matrix</code> <code>array</code> <p>A cost matrix (for classification tasks).</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClassificationTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Classification task type.\n    task_type : str\n        Name of the Classification task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the Classification task.\n    target_name : str\n        Name of the target variable.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the Classification task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the Classification task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure.\n    data_splits_url : str, default=None\n        URL of the data splits for the Classification task.\n    task_id : Union[int, None]\n        ID of the Classification task (if it already exists on OpenML).\n    class_labels : List of str, default=None\n        A list of class labels (for classification tasks).\n    cost_matrix : array, default=None\n        A cost matrix (for classification tasks).\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n        self.class_labels = class_labels\n        self.cost_matrix = cost_matrix\n\n        if cost_matrix is not None:\n            raise NotImplementedError(\"Costmatrix\")\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLClusteringTask","title":"<code>OpenMLClusteringTask</code>","text":"<p>               Bases: <code>OpenMLTask</code></p> <p>OpenML Clustering object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML clustering task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML clustering task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset used in clustering the task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML clustering task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure used in the clustering task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the clustering task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the clustering task.</p> <code>None</code> <code>target_name</code> <code>str</code> <p>Name of the target feature (class) that is not part of the feature set for the clustering task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLClusteringTask(OpenMLTask):\n    \"\"\"OpenML Clustering object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML clustering task.\n    task_type : str\n        Task type of the OpenML clustering task.\n    data_set_id : int\n        ID of the OpenML dataset used in clustering the task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    task_id : Union[int, None]\n        ID of the OpenML clustering task.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure used in the clustering task.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the clustering task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the clustering task.\n    target_name : str, default=None\n        Name of the target feature (class) that is not part of the\n        feature set for the clustering task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 17,\n        task_id: int | None = None,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        evaluation_measure: str | None = None,\n        target_name: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            evaluation_measure=evaluation_measure,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\"] = \"array\",\n    ) -&gt; np.ndarray | scipy.sparse.spmatrix:\n        ...\n\n    @overload\n    def get_X(self, dataset_format: Literal[\"dataframe\"]) -&gt; pd.DataFrame:\n        ...\n\n    def get_X(\n        self,\n        dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n    ) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        dataset = self.get_dataset()\n        data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n        return data\n\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        # Right now, it is not supported as a feature.\n        # Uncomment if it is supported on the server\n        # in the future.\n        # https://github.com/openml/OpenML/issues/925\n        \"\"\"\n        task_dict = task_container['oml:task_inputs']\n        if self.target_name is not None:\n            task_dict['oml:input'].append(\n                OrderedDict([\n                    ('@name', 'target_feature'),\n                    ('#text', self.target_name)\n                ])\n            )\n        \"\"\"\n        return super()._to_dict()\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLClusteringTask.get_X","title":"<code>get_X(dataset_format='array')</code>","text":"<pre><code>get_X(dataset_format: Literal['array'] = 'array') -&gt; np.ndarray | scipy.sparse.spmatrix\n</code></pre><pre><code>get_X(dataset_format: Literal['dataframe']) -&gt; pd.DataFrame\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X(\n    self,\n    dataset_format: Literal[\"array\", \"dataframe\"] = \"array\",\n) -&gt; np.ndarray | pd.DataFrame | scipy.sparse.spmatrix:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    dataset = self.get_dataset()\n    data, *_ = dataset.get_data(dataset_format=dataset_format, target=None)\n    return data\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLLearningCurveTask","title":"<code>OpenMLLearningCurveTask</code>","text":"<p>               Bases: <code>OpenMLClassificationTask</code></p> <p>OpenML Learning Curve object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the Learning Curve task.</p> required <code>task_type</code> <code>str</code> <p>Name of the Learning Curve task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the dataset that this task is associated with.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature in the dataset.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure to use for evaluating models.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Additional parameters for the estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the file containing the data splits for Learning Curve task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the Learning Curve task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure to use for evaluating models.</p> <code>None</code> <code>class_labels</code> <code>list of str</code> <p>Class labels for Learning Curve tasks.</p> <code>None</code> <code>cost_matrix</code> <code>numpy array</code> <p>Cost matrix for Learning Curve tasks.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLLearningCurveTask(OpenMLClassificationTask):\n    \"\"\"OpenML Learning Curve object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the Learning Curve task.\n    task_type : str\n        Name of the Learning Curve task.\n    data_set_id : int\n        ID of the dataset that this task is associated with.\n    target_name : str\n        Name of the target feature in the dataset.\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure to use for evaluating models.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure.\n    estimation_parameters : dict, default=None\n        Additional parameters for the estimation procedure.\n    data_splits_url : str, default=None\n        URL of the file containing the data splits for Learning Curve task.\n    task_id : Union[int, None]\n        ID of the Learning Curve task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure to use for evaluating models.\n    class_labels : list of str, default=None\n        Class labels for Learning Curve tasks.\n    cost_matrix : numpy array, default=None\n        Cost matrix for Learning Curve tasks.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 13,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n        class_labels: list[str] | None = None,\n        cost_matrix: np.ndarray | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n            class_labels=class_labels,\n            cost_matrix=cost_matrix,\n        )\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLRegressionTask","title":"<code>OpenMLRegressionTask</code>","text":"<p>               Bases: <code>OpenMLSupervisedTask</code></p> <p>OpenML Regression object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>Task type ID of the OpenML Regression task.</p> required <code>task_type</code> <code>str</code> <p>Task type of the OpenML Regression task.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature used in the Regression task.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the OpenML estimation procedure.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Parameters used by the OpenML estimation procedure.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the OpenML data splits for the Regression task.</p> <code>None</code> <code>task_id</code> <code>Union[int, None]</code> <p>ID of the OpenML Regression task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Evaluation measure used in the Regression task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLRegressionTask(OpenMLSupervisedTask):\n    \"\"\"OpenML Regression object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        Task type ID of the OpenML Regression task.\n    task_type : str\n        Task type of the OpenML Regression task.\n    data_set_id : int\n        ID of the OpenML dataset.\n    target_name : str\n        Name of the target feature used in the Regression task.\n    estimation_procedure_id : int, default=None\n        ID of the OpenML estimation procedure.\n    estimation_procedure_type : str, default=None\n        Type of the OpenML estimation procedure.\n    estimation_parameters : dict, default=None\n        Parameters used by the OpenML estimation procedure.\n    data_splits_url : str, default=None\n        URL of the OpenML data splits for the Regression task.\n    task_id : Union[int, None]\n        ID of the OpenML Regression task.\n    evaluation_measure : str, default=None\n        Evaluation measure used in the Regression task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 7,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n        evaluation_measure: str | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            target_name=target_name,\n            data_splits_url=data_splits_url,\n        )\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLSupervisedTask","title":"<code>OpenMLSupervisedTask</code>","text":"<p>               Bases: <code>OpenMLTask</code>, <code>ABC</code></p> <p>OpenML Supervised Classification object.</p> <p>Parameters:</p> Name Type Description Default <code>task_type_id</code> <code>TaskType</code> <p>ID of the task type.</p> required <code>task_type</code> <code>str</code> <p>Name of the task type.</p> required <code>data_set_id</code> <code>int</code> <p>ID of the OpenML dataset associated with the task.</p> required <code>target_name</code> <code>str</code> <p>Name of the target feature (the class variable).</p> required <code>estimation_procedure_id</code> <code>int</code> <p>ID of the estimation procedure for the task.</p> <code>None</code> <code>estimation_procedure_type</code> <code>str</code> <p>Type of the estimation procedure for the task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict</code> <p>Estimation parameters for the task.</p> <code>None</code> <code>evaluation_measure</code> <code>str</code> <p>Name of the evaluation measure for the task.</p> <code>None</code> <code>data_splits_url</code> <code>str</code> <p>URL of the data splits for the task.</p> <code>None</code> <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLSupervisedTask(OpenMLTask, ABC):\n    \"\"\"OpenML Supervised Classification object.\n\n    Parameters\n    ----------\n    task_type_id : TaskType\n        ID of the task type.\n    task_type : str\n        Name of the task type.\n    data_set_id : int\n        ID of the OpenML dataset associated with the task.\n    target_name : str\n        Name of the target feature (the class variable).\n    estimation_procedure_id : int, default=None\n        ID of the estimation procedure for the task.\n    estimation_procedure_type : str, default=None\n        Type of the estimation procedure for the task.\n    estimation_parameters : dict, default=None\n        Estimation parameters for the task.\n    evaluation_measure : str, default=None\n        Name of the evaluation measure for the task.\n    data_splits_url : str, default=None\n        URL of the data splits for the task.\n    task_id: Union[int, None]\n        Refers to the unique identifier of task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        target_name: str,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n        task_id: int | None = None,\n    ):\n        super().__init__(\n            task_id=task_id,\n            task_type_id=task_type_id,\n            task_type=task_type,\n            data_set_id=data_set_id,\n            estimation_procedure_id=estimation_procedure_id,\n            estimation_procedure_type=estimation_procedure_type,\n            estimation_parameters=estimation_parameters,\n            evaluation_measure=evaluation_measure,\n            data_splits_url=data_splits_url,\n        )\n\n        self.target_name = target_name\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | scipy.sparse.spmatrix,\n        np.ndarray | None,\n    ]:\n        ...\n\n    @overload\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\"]\n    ) -&gt; tuple[\n        pd.DataFrame,\n        pd.Series | pd.DataFrame | None,\n    ]:\n        ...\n\n    # TODO(eddiebergman): Do all OpenMLSupervisedTask have a `y`?\n    def get_X_and_y(\n        self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n    ) -&gt; tuple[\n        np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n        np.ndarray | pd.Series | pd.DataFrame | None,\n    ]:\n        \"\"\"Get data associated with the current task.\n\n        Parameters\n        ----------\n        dataset_format : str\n            Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n            for possible options.\n\n        Returns\n        -------\n        tuple - X and y\n\n        \"\"\"\n        # TODO: [0.15]\n        if dataset_format == \"array\":\n            warnings.warn(\n                \"Support for `dataset_format='array'` will be removed in 0.15,\"\n                \"start using `dataset_format='dataframe' to ensure your code \"\n                \"will continue to work. You can use the dataframe's `to_numpy` \"\n                \"function to continue using numpy arrays.\",\n                category=FutureWarning,\n                stacklevel=2,\n            )\n        dataset = self.get_dataset()\n        if self.task_type_id not in (\n            TaskType.SUPERVISED_CLASSIFICATION,\n            TaskType.SUPERVISED_REGRESSION,\n            TaskType.LEARNING_CURVE,\n        ):\n            raise NotImplementedError(self.task_type)\n\n        X, y, _, _ = dataset.get_data(\n            dataset_format=dataset_format,\n            target=self.target_name,\n        )\n        return X, y\n\n    def _to_dict(self) -&gt; dict[str, dict]:\n        task_container = super()._to_dict()\n        oml_input = task_container[\"oml:task_inputs\"][\"oml:input\"]  # type: ignore\n        assert isinstance(oml_input, list)\n\n        oml_input.append({\"@name\": \"target_feature\", \"#text\": self.target_name})\n        return task_container\n\n    @property\n    def estimation_parameters(self) -&gt; dict[str, str] | None:\n        \"\"\"Return the estimation parameters for the task.\"\"\"\n        warnings.warn(\n            \"The estimation_parameters attribute will be \"\n            \"deprecated in the future, please use \"\n            \"estimation_procedure['parameters'] instead\",\n            PendingDeprecationWarning,\n            stacklevel=2,\n        )\n        return self.estimation_procedure[\"parameters\"]\n\n    @estimation_parameters.setter\n    def estimation_parameters(self, est_parameters: dict[str, str] | None) -&gt; None:\n        self.estimation_procedure[\"parameters\"] = est_parameters\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLSupervisedTask.estimation_parameters","title":"<code>estimation_parameters: dict[str, str] | None</code>  <code>property</code> <code>writable</code>","text":"<p>Return the estimation parameters for the task.</p>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLSupervisedTask.get_X_and_y","title":"<code>get_X_and_y(dataset_format='array')</code>","text":"<pre><code>get_X_and_y(dataset_format: Literal['array'] = 'array') -&gt; tuple[np.ndarray | scipy.sparse.spmatrix, np.ndarray | None]\n</code></pre><pre><code>get_X_and_y(dataset_format: Literal['dataframe']) -&gt; tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]\n</code></pre> <p>Get data associated with the current task.</p> <p>Parameters:</p> Name Type Description Default <code>dataset_format</code> <code>str</code> <p>Data structure of the returned data. See :meth:<code>openml.datasets.OpenMLDataset.get_data</code> for possible options.</p> <code>'array'</code> <p>Returns:</p> Type Description <code>tuple - X and y</code> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_X_and_y(\n    self, dataset_format: Literal[\"dataframe\", \"array\"] = \"array\"\n) -&gt; tuple[\n    np.ndarray | pd.DataFrame | scipy.sparse.spmatrix,\n    np.ndarray | pd.Series | pd.DataFrame | None,\n]:\n    \"\"\"Get data associated with the current task.\n\n    Parameters\n    ----------\n    dataset_format : str\n        Data structure of the returned data. See :meth:`openml.datasets.OpenMLDataset.get_data`\n        for possible options.\n\n    Returns\n    -------\n    tuple - X and y\n\n    \"\"\"\n    # TODO: [0.15]\n    if dataset_format == \"array\":\n        warnings.warn(\n            \"Support for `dataset_format='array'` will be removed in 0.15,\"\n            \"start using `dataset_format='dataframe' to ensure your code \"\n            \"will continue to work. You can use the dataframe's `to_numpy` \"\n            \"function to continue using numpy arrays.\",\n            category=FutureWarning,\n            stacklevel=2,\n        )\n    dataset = self.get_dataset()\n    if self.task_type_id not in (\n        TaskType.SUPERVISED_CLASSIFICATION,\n        TaskType.SUPERVISED_REGRESSION,\n        TaskType.LEARNING_CURVE,\n    ):\n        raise NotImplementedError(self.task_type)\n\n    X, y, _, _ = dataset.get_data(\n        dataset_format=dataset_format,\n        target=self.target_name,\n    )\n    return X, y\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask","title":"<code>OpenMLTask</code>","text":"<p>               Bases: <code>OpenMLBase</code></p> <p>OpenML Task object.</p> <p>Parameters:</p> Name Type Description Default <code>task_id</code> <code>int | None</code> <p>Refers to the unique identifier of OpenML task.</p> required <code>task_type_id</code> <code>TaskType</code> <p>Refers to the type of OpenML task.</p> required <code>task_type</code> <code>str</code> <p>Refers to the OpenML task.</p> required <code>data_set_id</code> <code>int</code> <p>Refers to the data.</p> required <code>estimation_procedure_id</code> <code>int</code> <p>Refers to the type of estimates used.</p> <code>1</code> <code>estimation_procedure_type</code> <code>str | None</code> <p>Refers to the type of estimation procedure used for the OpenML task.</p> <code>None</code> <code>estimation_parameters</code> <code>dict[str, str] | None</code> <p>Estimation parameters used for the OpenML task.</p> <code>None</code> <code>evaluation_measure</code> <code>str | None</code> <p>Refers to the evaluation measure.</p> <code>None</code> <code>data_splits_url</code> <code>str | None</code> <p>Refers to the URL of the data splits used for the OpenML task.</p> <code>None</code> Source code in <code>openml/tasks/task.py</code> <pre><code>class OpenMLTask(OpenMLBase):\n    \"\"\"OpenML Task object.\n\n    Parameters\n    ----------\n    task_id: Union[int, None]\n        Refers to the unique identifier of OpenML task.\n    task_type_id: TaskType\n        Refers to the type of OpenML task.\n    task_type: str\n        Refers to the OpenML task.\n    data_set_id: int\n        Refers to the data.\n    estimation_procedure_id: int\n        Refers to the type of estimates used.\n    estimation_procedure_type: str, default=None\n        Refers to the type of estimation procedure used for the OpenML task.\n    estimation_parameters: [Dict[str, str]], default=None\n        Estimation parameters used for the OpenML task.\n    evaluation_measure: str, default=None\n        Refers to the evaluation measure.\n    data_splits_url: str, default=None\n        Refers to the URL of the data splits used for the OpenML task.\n    \"\"\"\n\n    def __init__(  # noqa: PLR0913\n        self,\n        task_id: int | None,\n        task_type_id: TaskType,\n        task_type: str,\n        data_set_id: int,\n        estimation_procedure_id: int = 1,\n        estimation_procedure_type: str | None = None,\n        estimation_parameters: dict[str, str] | None = None,\n        evaluation_measure: str | None = None,\n        data_splits_url: str | None = None,\n    ):\n        self.task_id = int(task_id) if task_id is not None else None\n        self.task_type_id = task_type_id\n        self.task_type = task_type\n        self.dataset_id = int(data_set_id)\n        self.evaluation_measure = evaluation_measure\n        self.estimation_procedure: _EstimationProcedure = {\n            \"type\": estimation_procedure_type,\n            \"parameters\": estimation_parameters,\n            \"data_splits_url\": data_splits_url,\n        }\n        self.estimation_procedure_id = estimation_procedure_id\n        self.split: OpenMLSplit | None = None\n\n    @classmethod\n    def _entity_letter(cls) -&gt; str:\n        return \"t\"\n\n    @property\n    def id(self) -&gt; int | None:\n        \"\"\"Return the OpenML ID of this task.\"\"\"\n        return self.task_id\n\n    def _get_repr_body_fields(self) -&gt; Sequence[tuple[str, str | int | list[str]]]:\n        \"\"\"Collect all information to display in the __repr__ body.\"\"\"\n        base_server_url = openml.config.get_server_base_url()\n        fields: dict[str, Any] = {\n            \"Task Type Description\": f\"{base_server_url}/tt/{self.task_type_id}\"\n        }\n        if self.task_id is not None:\n            fields[\"Task ID\"] = self.task_id\n            fields[\"Task URL\"] = self.openml_url\n        if self.evaluation_measure is not None:\n            fields[\"Evaluation Measure\"] = self.evaluation_measure\n        if self.estimation_procedure is not None:\n            fields[\"Estimation Procedure\"] = self.estimation_procedure[\"type\"]\n\n        # TODO(eddiebergman): Subclasses could advertise/provide this, instead of having to\n        # have the base class know about it's subclasses.\n        target_name = getattr(self, \"target_name\", None)\n        if target_name is not None:\n            fields[\"Target Feature\"] = target_name\n\n            class_labels = getattr(self, \"class_labels\", None)\n            if class_labels is not None:\n                fields[\"# of Classes\"] = len(class_labels)\n\n            if hasattr(self, \"cost_matrix\"):\n                fields[\"Cost Matrix\"] = \"Available\"\n\n        # determines the order in which the information will be printed\n        order = [\n            \"Task Type Description\",\n            \"Task ID\",\n            \"Task URL\",\n            \"Estimation Procedure\",\n            \"Evaluation Measure\",\n            \"Target Feature\",\n            \"# of Classes\",\n            \"Cost Matrix\",\n        ]\n        return [(key, fields[key]) for key in order if key in fields]\n\n    def get_dataset(self) -&gt; datasets.OpenMLDataset:\n        \"\"\"Download dataset associated with task.\"\"\"\n        return datasets.get_dataset(self.dataset_id)\n\n    def get_train_test_split_indices(\n        self,\n        fold: int = 0,\n        repeat: int = 0,\n        sample: int = 0,\n    ) -&gt; tuple[np.ndarray, np.ndarray]:\n        \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n        # Replace with retrieve from cache\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.get(repeat=repeat, fold=fold, sample=sample)\n\n    def _download_split(self, cache_file: Path) -&gt; None:\n        # TODO(eddiebergman): Not sure about this try to read and error approach\n        try:\n            with cache_file.open(encoding=\"utf8\"):\n                pass\n        except OSError:\n            split_url = self.estimation_procedure[\"data_splits_url\"]\n            openml._api_calls._download_text_file(\n                source=str(split_url),\n                output_path=str(cache_file),\n            )\n\n    def download_split(self) -&gt; OpenMLSplit:\n        \"\"\"Download the OpenML split for a given task.\"\"\"\n        # TODO(eddiebergman): Can this every be `None`?\n        assert self.task_id is not None\n        cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n        cached_split_file = cache_dir / \"datasplits.arff\"\n\n        try:\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n        except OSError:\n            # Next, download and cache the associated split file\n            self._download_split(cached_split_file)\n            split = OpenMLSplit._from_arff_file(cached_split_file)\n\n        return split\n\n    def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n        \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n        if self.split is None:\n            self.split = self.download_split()\n\n        return self.split.repeats, self.split.folds, self.split.samples\n\n    # TODO(eddiebergman): Really need some better typing on all this\n    def _to_dict(self) -&gt; dict[str, dict[str, int | str | list[dict[str, Any]]]]:\n        \"\"\"Creates a dictionary representation of self in a string format (for XML parsing).\"\"\"\n        oml_input = [\n            {\"@name\": \"source_data\", \"#text\": str(self.dataset_id)},\n            {\"@name\": \"estimation_procedure\", \"#text\": str(self.estimation_procedure_id)},\n        ]\n        if self.evaluation_measure is not None:  #\n            oml_input.append({\"@name\": \"evaluation_measures\", \"#text\": self.evaluation_measure})\n\n        return {\n            \"oml:task_inputs\": {\n                \"@xmlns:oml\": \"http://openml.org/openml\",\n                \"oml:task_type_id\": self.task_type_id.value,  # This is an int from the enum?\n                \"oml:input\": oml_input,\n            }\n        }\n\n    def _parse_publish_response(self, xml_response: dict) -&gt; None:\n        \"\"\"Parse the id from the xml_response and assign it to self.\"\"\"\n        self.task_id = int(xml_response[\"oml:upload_task\"][\"oml:id\"])\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.id","title":"<code>id: int | None</code>  <code>property</code>","text":"<p>Return the OpenML ID of this task.</p>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.download_split","title":"<code>download_split()</code>","text":"<p>Download the OpenML split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def download_split(self) -&gt; OpenMLSplit:\n    \"\"\"Download the OpenML split for a given task.\"\"\"\n    # TODO(eddiebergman): Can this every be `None`?\n    assert self.task_id is not None\n    cache_dir = _create_cache_directory_for_id(\"tasks\", self.task_id)\n    cached_split_file = cache_dir / \"datasplits.arff\"\n\n    try:\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n    except OSError:\n        # Next, download and cache the associated split file\n        self._download_split(cached_split_file)\n        split = OpenMLSplit._from_arff_file(cached_split_file)\n\n    return split\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.get_dataset","title":"<code>get_dataset()</code>","text":"<p>Download dataset associated with task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_dataset(self) -&gt; datasets.OpenMLDataset:\n    \"\"\"Download dataset associated with task.\"\"\"\n    return datasets.get_dataset(self.dataset_id)\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.get_split_dimensions","title":"<code>get_split_dimensions()</code>","text":"<p>Get the (repeats, folds, samples) of the split for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_split_dimensions(self) -&gt; tuple[int, int, int]:\n    \"\"\"Get the (repeats, folds, samples) of the split for a given task.\"\"\"\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.repeats, self.split.folds, self.split.samples\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.OpenMLTask.get_train_test_split_indices","title":"<code>get_train_test_split_indices(fold=0, repeat=0, sample=0)</code>","text":"<p>Get the indices of the train and test splits for a given task.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>def get_train_test_split_indices(\n    self,\n    fold: int = 0,\n    repeat: int = 0,\n    sample: int = 0,\n) -&gt; tuple[np.ndarray, np.ndarray]:\n    \"\"\"Get the indices of the train and test splits for a given task.\"\"\"\n    # Replace with retrieve from cache\n    if self.split is None:\n        self.split = self.download_split()\n\n    return self.split.get(repeat=repeat, fold=fold, sample=sample)\n</code></pre>"},{"location":"reference/tasks/task/#openml.tasks.task.TaskType","title":"<code>TaskType</code>","text":"<p>               Bases: <code>Enum</code></p> <p>Possible task types as defined in OpenML.</p> Source code in <code>openml/tasks/task.py</code> <pre><code>class TaskType(Enum):\n    \"\"\"Possible task types as defined in OpenML.\"\"\"\n\n    SUPERVISED_CLASSIFICATION = 1\n    SUPERVISED_REGRESSION = 2\n    LEARNING_CURVE = 3\n    SUPERVISED_DATASTREAM_CLASSIFICATION = 4\n    CLUSTERING = 5\n    MACHINE_LEARNING_CHALLENGE = 6\n    SURVIVAL_ANALYSIS = 7\n    SUBGROUP_DISCOVERY = 8\n    MULTITASK_REGRESSION = 9\n</code></pre>"},{"location":"services/","title":"services","text":"<p>Overview of all OpenML components including a docker-compose to run OpenML services locally</p>"},{"location":"services/#overview","title":"Overview","text":""},{"location":"services/#prerequisites","title":"Prerequisites","text":"<ul> <li>Linux/MacOS/Windows (should all work)</li> <li>Docker </li> <li>Docker Compose version 2.21.0 or higher</li> </ul>"},{"location":"services/#usage","title":"Usage","text":"<p>You run all OpenML services locally using <pre><code>docker compose --profile all up -d\n</code></pre> Stop it again using  <pre><code>docker compose --profile all down\n</code></pre></p>"},{"location":"services/#profiles","title":"Profiles","text":"<p>You can use different profiles:</p> <ul> <li><code>[no profile]</code>: databases</li> <li><code>\"elasticsearch\"</code>: databases + elasticsearch</li> <li><code>\"rest-api\"</code>: databases + elasticsearch + REST API</li> <li><code>\"frontend\"</code>: databases + elasticsearch + REST API + frontend + email-server</li> <li><code>\"minio\"</code>: database + elasticsearch + REST APP + MinIO + parquet and croissant conversion</li> <li><code>\"all\"</code>: everything</li> </ul> <p>Usage examples: <pre><code>docker compose --profile all up -d       # all services\ndocker compose up -d                     # only the database\ndocker compose --profile frontend up -d  # Frontend, rest-api, elasticsearch and database\n</code></pre> Use the same profile for your <code>down</code> command.</p>"},{"location":"services/#known-issues","title":"Known issues","text":"<p>See the Github Issue list for the known issues.</p>"},{"location":"services/#debugging","title":"Debugging","text":"<p>Some usefull commands: <pre><code>docker logs openml-php-rest-api -f              # tail the logs of the php rest api\ndocker exec -it openml-php-rest-api /bin/bash   # go into the php rest api container\n./scripts/connect_db.sql                        # access the database\n</code></pre></p>"},{"location":"services/#endpoints","title":"Endpoints","text":"<p>[!TIP] If you change any port, make sure to change it for all services! The elasticsearch config, for instance, needs to know the port of the frontend (for CORS).</p> <p>When you spin up the docker-compose, you'll get these endpoints: - Frontend: localhost:5000 - Database: localhost:3306, filled with test data. - ElasticSearch: localhost:9200, filled with test data. - Rest API: localhost:8080 - Minio: console at localhost:9001, filled with test data.</p>"},{"location":"services/#credentials","title":"Credentials","text":"<p>The credentials for the database can be found in <code>config/database/.env</code>, for minio in <code>config/minio/.env</code>, etc.</p>"},{"location":"services/#emails","title":"Emails","text":"<p>The email-server is used for emails from the frontend. For example, if you create a new user, an  email is send to the user. All outgoing emails are rerouted to catchall@example.com. You can see  the messages in <code>config/email-server/messages</code>. Note that some of the urls in the emails need to  be slightly altered to use them in the test setup: change https to http.</p>"},{"location":"services/#development","title":"Development","text":""},{"location":"services/#php-parquet-and-croissant-converter","title":"PHP, Parquet and Croissant converter","text":"<p>If you want to do local development on containers that are part of the docker-compose, you want those containers to change based on your code. You should have the relevant code somewhere on your system, you only need to tell the docker-compose where to find it. You can do so by setting environment variables. </p> <p>Create a <code>.env</code> file inside this directory, and set:</p>"},{"location":"services/#php","title":"PHP","text":"<pre><code>PHP_CODE_DIR=/path/to/OpenML                  # Root of https://github.com/openml/OpenML on your computer\nPHP_CODE_VAR_WWW_OPENML=/var/www/openml       # Always set this to /var/www/openml. Leave empty if you leave PHP_CODE_DIR empty\n</code></pre> <p>Make sure to create <code>openml_OS/config/BASE_CONFIG.php</code> in your local <code>$PHP_CODE_DIR</code>. The correct configuration can be found in <code>config/php.env</code>. Run docker compose with profile <code>rest-api</code>.</p>"},{"location":"services/#parquet","title":"Parquet","text":"<pre><code>ARFF_TO_PQ_CODE_DIR=/path/to/minio-data       # Root of https://github.com/openml-labs/minio-data on your computer\nARFF_TO_PQ_APP=/app                           # Always set this to /app. Leave empty if you leave ARFF_TO_PQ_CODE_DIR empty\n</code></pre>"},{"location":"services/#croissant","title":"Croissant","text":"<pre><code>CROISSANT_CODE_DIR=/path/to/openml-croissant/python  # Python directory of https://github.com/openml/openml-croissant on your computer\nCROISSANT_APP=/app                                   # Always set this to /app. Leave empty if you leave CROISSANT_CODE_DIR empty\n</code></pre>"},{"location":"services/#frontend","title":"Frontend","text":"<pre><code>FRONTEND_CODE_DIR=/path/to/openml.org        # Python directory of https://github.com/openml/openml.org on your computer\nFRONTEND_APP=/app                            # Always set this to /app. Leave empty if you leave FRONTEND_CODE_DIR empty\n</code></pre>"},{"location":"services/#other-services","title":"Other services","text":"<p>If you want to develop a service that depends on any of the services in this docker-compose, just bring up this docker-compose and point your service to the correct endpoints.</p>"},{"location":"tensorflow/","title":"Tensorflow extension for OpenML python","text":"<p>Tensorflow extension for openml-python API.</p>"},{"location":"tensorflow/#installation-instructions","title":"Installation Instructions:","text":"<p><code>pip install openml-tensorflow</code></p> <p>PyPi link https://pypi.org/project/openml-keras/</p>"},{"location":"tensorflow/#usage","title":"Usage","text":"<p>Import openML libraries <pre><code>import openml\nimport openml_tensorflow\n</code></pre> Create  and compile a keras model <pre><code>model = tensorflow.keras.models.Sequential([\n    tensorflow.keras.layers.BatchNormalization(),\n    tensorflow.keras.layers.Dense(units=1024, activation=tensorflow.keras.activations.relu),\n    tensorflow.keras.layers.Dropout(rate=0.4),\n    tensorflow.keras.layers.Dense(units=2, activation=tensorflow.keras.activations.softmax),\n])\n\n# We will compile using the Adam optimizer while targeting accuracy.\nmodel.compile(optimizer='adam',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n</code></pre> Download the task from openML and run the model on task. <pre><code>task = openml.tasks.get_task(3573)\nrun = openml.runs.run_model_on_task(model, task, avoid_duplicate_runs=False)\nrun.publish()\nprint('URL for run: %s/run/%d' % (openml.config.server, run.run_id))\n</code></pre></p> <p>This library is currently under development, please report any bugs or feature reuest in issues section.</p>"}]}
\ No newline at end of file
diff --git a/services/index.html b/services/index.html
index 2ed2f69..8012114 100644
--- a/services/index.html
+++ b/services/index.html
@@ -373,6 +373,23 @@
   
   
   
+    <li class="md-tabs__item">
+      <a href="../tensorflow/" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
     
     
       <li class="md-tabs__item">
@@ -4467,6 +4484,26 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../tensorflow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
     
     
       
@@ -4484,7 +4521,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
         
           
           
@@ -4500,14 +4537,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_8" id="__nav_8_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_8">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
             <span class="md-nav__icon md-icon"></span>
             Julia
           </label>
@@ -4573,7 +4610,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
         
           
           
@@ -4589,14 +4626,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10" id="__nav_10_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
             <span class="md-nav__icon md-icon"></span>
             APIv2
           </label>
@@ -4652,7 +4689,7 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_10_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
         
           
           
@@ -4668,14 +4705,14 @@
             </a>
             
               
-              <label class="md-nav__link " for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
                 <span class="md-nav__icon md-icon"></span>
               </label>
             
           </div>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_10_3">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
             <span class="md-nav__icon md-icon"></span>
             Contributing
           </label>
@@ -5132,7 +5169,7 @@ <h3 id="other-services">Other services<a class="headerlink" href="#other-service
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
@@ -5142,7 +5179,7 @@ <h3 id="other-services">Other services<a class="headerlink" href="#other-service
     <span class="md-icon" title="Created">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 30, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
   </span>
 
     
diff --git a/sitemap.xml b/sitemap.xml
index 5a732af..68349fd 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,590 +2,594 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://docs.openml.org/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/installation/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/migration/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/contributing/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/contributing/contributing/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/contributing/documentation/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/contributing/project_overview/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/apiv2/contributing/tests/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/benchmark/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/benchmark/automl/AutoML-Benchmark/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/benchmark/automl/basic_example/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/benchmark/automl/benchmark_on_openml/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/benchmark/automl/important_params/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/benchmark/automl/specific_task_fold_example/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/authentication/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/benchmarking/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/data/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/flows/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/runs/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/sharing/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/tagging/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/concepts/tasks/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/OpenML-Docs/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/Style/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/resources/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/backend/API-development/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/backend/Datasets/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/backend/Java-App/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/backend/Local-Installation/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/clients/Client-API-Standards/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/clients/Rest/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/clients/metadata_definition/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/website/Dash/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/website/Flask/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/website/React/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/contributing/website/Website/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/Java/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/MOA/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/Python_extensions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/Rest/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/Weka/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/mlr/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/Scikit-learn/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/Scikit-learn/basic_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/ecosystem/Scikit-learn/datasets_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/SUMMARY/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/20_basic/introduction_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/20_basic/simple_datasets_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/20_basic/simple_flows_and_runs_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/20_basic/simple_suites_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/configure_logging/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/create_upload_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/custom_flow_/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/datasets_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/fetch_evaluations_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/fetch_runtimes_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/flow_id_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/flows_and_runs_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/plot_svm_hyperparameters_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/run_setup_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/study_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/suites_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/task_manual_iteration_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/30_extended/tasks_tutorial/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/40_paper/2015_neurips_feurer_example/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/40_paper/2018_ida_strang_example/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/40_paper/2018_kdd_rijn_example/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/examples/40_paper/2018_neurips_perrone_example/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/help/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/intro/Governance/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/intro/showcase/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/intro/terms/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/julia/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/notebooks/getting_started/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/CONTRIBUTING/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/ISSUE_TEMPLATE/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/PULL_REQUEST_TEMPLATE/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/contributing/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/extensions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/progress/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/usage/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/docker/readme/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/docs/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/docs/contributing/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/docs/extensions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/docs/progress/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/python/docs/usage/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/Limitations%20of%20the%20API/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/Philosophy%20behind%20the%20API%20Design/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/API%20reference/Callbacks/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/API%20reference/Custom%20Datasets/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/API%20reference/Metrics/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/API%20reference/OpenML%20Connection/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/API%20reference/Trainer/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/Examples/Create%20Dataset%20and%20Task/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/Examples/Image%20Classification%20Task/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/Examples/Sequential%20Classification%20Task/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/pytorch/Examples/Tabular%20Classification/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/r/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/SUMMARY/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/__version__/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/_api_calls/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/base/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/cli/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/config/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/exceptions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/testing/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/utils/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/datasets/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/datasets/data_feature/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/datasets/dataset/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/datasets/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/evaluations/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/evaluations/evaluation/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/evaluations/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/extensions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/extensions/extension_interface/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/extensions/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/extensions/sklearn/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/extensions/sklearn/extension/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/flows/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/flows/flow/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/flows/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/runs/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/runs/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/runs/run/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/runs/trace/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/setups/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/setups/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/setups/setup/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/study/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/study/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/study/study/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/tasks/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/tasks/functions/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/tasks/split/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/reference/tasks/task/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
     </url>
     <url>
          <loc>https://docs.openml.org/services/</loc>
-         <lastmod>2024-10-30</lastmod>
+         <lastmod>2024-10-31</lastmod>
+    </url>
+    <url>
+         <loc>https://docs.openml.org/tensorflow/</loc>
+         <lastmod>2024-10-31</lastmod>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index eaa1637..0d6a992 100644
Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ
diff --git a/tensorflow/index.html b/tensorflow/index.html
new file mode 100644
index 0000000..59965ad
--- /dev/null
+++ b/tensorflow/index.html
@@ -0,0 +1,5140 @@
+
+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+    
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      
+      
+      
+        <link rel="canonical" href="https://docs.openml.org/tensorflow/">
+      
+      
+        <link rel="prev" href="../pytorch/Philosophy%20behind%20the%20API%20Design/">
+      
+      
+        <link rel="next" href="../julia/">
+      
+      
+      <link rel="icon" href="../assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.40">
+    
+    
+      
+        <title>TensorFlow - Open Machine Learning</title>
+      
+    
+    
+      <link rel="stylesheet" href="../assets/stylesheets/main.8c3ca2c6.min.css">
+      
+        
+        <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
+      
+      
+
+
+    
+    
+      
+    
+    
+      
+        
+        
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+      
+    
+    
+      <link rel="stylesheet" href="../assets/_mkdocstrings.css">
+    
+      <link rel="stylesheet" href="../css/extra.css">
+    
+    <script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+    
+      
+
+    
+    
+    
+  </head>
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
+  
+    
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+      
+        
+        <a href="#tensorflow-extension-for-openml-python" class="md-skip">
+          Skip to content
+        </a>
+      
+    </div>
+    <div data-md-component="announce">
+      
+    </div>
+    
+    
+      
+
+  
+
+<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href=".." title="Open Machine Learning" class="md-header__button md-logo" aria-label="Open Machine Learning" data-md-component="logo">
+      
+  <img src="../img/openml.png" alt="logo">
+
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            Open Machine Learning
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            
+              TensorFlow
+            
+          </span>
+        </div>
+      </div>
+    </div>
+    
+      
+        <form class="md-header__option" data-md-component="palette">
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label=""  type="radio" name="__palette" id="__palette_0">
+    
+      <label class="md-header__button md-icon" title="" for="__palette_1" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m14.3 16-.7-2h-3.2l-.7 2H7.8L11 7h2l3.2 9zM20 8.69V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12zm-9.15 3.96h2.3L12 9z"/></svg>
+      </label>
+    
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label=""  type="radio" name="__palette" id="__palette_1">
+    
+      <label class="md-header__button md-icon" title="" for="__palette_2" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
+      </label>
+    
+  
+    
+    
+    
+    <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo"  aria-label=""  type="radio" name="__palette" id="__palette_2">
+    
+      <label class="md-header__button md-icon" title="" for="__palette_0" hidden>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
+      </label>
+    
+  
+</form>
+      
+    
+    
+      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
+    
+    
+    
+      <label class="md-header__button md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+      </label>
+      <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+      
+        <div class="md-search__suggest" data-md-component="search-suggest"></div>
+      
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+    
+    
+      <div class="md-header__source">
+        <a href="https://github.com/openml/docs/" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
+  </div>
+  <div class="md-source__repository">
+    openml/docs
+  </div>
+</a>
+      </div>
+    
+  </nav>
+  
+    
+      
+<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
+  <div class="md-grid">
+    <ul class="md-tabs__list">
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href=".." class="md-tabs__link">
+          
+  
+    
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 19c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m0-18c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2M6 16c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m-3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m12 12c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m-3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2"/></svg>
+    
+  
+  OpenML
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../benchmark/" class="md-tabs__link">
+          
+  
+    
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M0 256a256 256 0 1 1 512 0 256 256 0 1 1-512 0M288 96a32 32 0 1 0-64 0 32 32 0 1 0 64 0m-32 320c35.3 0 64-28.7 64-64 0-17.4-6.9-33.1-18.1-44.6L366 161.7c5.3-12.1-.2-26.3-12.3-31.6s-26.3.2-31.6 12.3L257.9 288H256c-35.3 0-64 28.7-64 64s28.7 64 64 64m-80-272a32 32 0 1 0-64 0 32 32 0 1 0 64 0M96 288a32 32 0 1 0 0-64 32 32 0 1 0 0 64m352-32a32 32 0 1 0-64 0 32 32 0 1 0 64 0"/></svg>
+    
+  
+  Benchmarking
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../ecosystem/" class="md-tabs__link">
+          
+  
+    
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M512 32c0 113.6-84.6 207.5-194.2 222-7.1-53.4-30.6-101.6-65.3-139.3C290.8 46.3 364 0 448 0h32c17.7 0 32 14.3 32 32M0 96c0-17.7 14.3-32 32-32h32c123.7 0 224 100.3 224 224v192c0 17.7-14.3 32-32 32s-32-14.3-32-32V320C100.3 320 0 219.7 0 96"/></svg>
+    
+  
+  Ecosystem
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../contributing/" class="md-tabs__link">
+          
+  
+    
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M256 64a64 64 0 1 0-128 0 64 64 0 1 0 128 0M152.9 169.3c-23.7-8.4-44.5-24.3-58.8-45.8L74.6 94.2C64.8 79.5 45 75.6 30.2 85.4s-18.7 29.7-8.9 44.4L40.9 159c18.1 27.1 42.8 48.4 71.1 62.4V480c0 17.7 14.3 32 32 32s32-14.3 32-32v-96h32v96c0 17.7 14.3 32 32 32s32-14.3 32-32V221.6c29.1-14.2 54.4-36.2 72.7-64.2l18.2-27.9c9.6-14.8 5.4-34.6-9.4-44.3s-34.6-5.5-44.3 9.4L291 122.4c-21.8 33.4-58.9 53.6-98.8 53.6-12.6 0-24.9-2-36.6-5.8-.9-.3-1.8-.7-2.7-.9"/></svg>
+    
+  
+  Contributing
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../help/" class="md-tabs__link">
+        
+  
+    
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m15.07 11.25-.9.92C13.45 12.89 13 13.5 13 15h-2v-.5c0-1.11.45-2.11 1.17-2.83l1.24-1.26c.37-.36.59-.86.59-1.41a2 2 0 0 0-2-2 2 2 0 0 0-2 2H8a4 4 0 0 1 4-4 4 4 0 0 1 4 4 3.2 3.2 0 0 1-.93 2.25M13 19h-2v-2h2M12 2A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10c0-5.53-4.5-10-10-10"/></svg>
+    
+  
+  Questions
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../python/" class="md-tabs__link">
+        
+  
+    
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9 3v15h3V3zm3 2 4 13 3-1-4-13zM5 5v13h3V5zM3 19v2h18v-2z"/></svg>
+    
+  
+  Python
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../pytorch/" class="md-tabs__link">
+          
+  
+    
+  
+  PyTorch
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+    
+  
+  
+    <li class="md-tabs__item md-tabs__item--active">
+      <a href="./" class="md-tabs__link">
+        
+  
+    
+  
+  TensorFlow
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../julia/" class="md-tabs__link">
+          
+  
+    
+  
+  Julia
+
+        </a>
+      </li>
+    
+  
+
+      
+        
+  
+  
+  
+    <li class="md-tabs__item">
+      <a href="../r/" class="md-tabs__link">
+        
+  
+    
+  
+  R
+
+      </a>
+    </li>
+  
+
+      
+        
+  
+  
+  
+    
+    
+      <li class="md-tabs__item">
+        <a href="../apiv2/" class="md-tabs__link">
+          
+  
+    
+  
+  APIv2
+
+        </a>
+      </li>
+    
+  
+
+      
+    </ul>
+  </div>
+</nav>
+    
+  
+</header>
+    
+    <div class="md-container" data-md-component="container">
+      
+      
+        
+      
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+          
+            
+              
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+
+  
+
+
+<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href=".." title="Open Machine Learning" class="md-nav__button md-logo" aria-label="Open Machine Learning" data-md-component="logo">
+      
+  <img src="../img/openml.png" alt="logo">
+
+    </a>
+    Open Machine Learning
+  </label>
+  
+    <div class="md-nav__source">
+      <a href="https://github.com/openml/docs/" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
+  </div>
+  <div class="md-source__repository">
+    openml/docs
+  </div>
+</a>
+    </div>
+  
+  <ul class="md-nav__list" data-md-scrollfix>
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href=".." class="md-nav__link ">
+              
+  
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 19c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m0-18c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2M6 16c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m-3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m12 12c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2m-3-6c1.1 0 2 .9 2 2s-.9 2-2 2-2-.9-2-2 .9-2 2-2"/></svg>
+  
+  <span class="md-ellipsis">
+    OpenML
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_1" id="__nav_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_1">
+            <span class="md-nav__icon md-icon"></span>
+            OpenML
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../notebooks/getting_started/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Getting Started
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_1_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../concepts/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Concepts
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_1_3" id="__nav_1_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_1_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_1_3">
+            <span class="md-nav__icon md-icon"></span>
+            Concepts
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/data/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Data
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/tasks/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Tasks
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/flows/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Flows
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/runs/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Runs
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/benchmarking/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Collections & Benchmarks
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/tagging/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Tagging
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/authentication/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Authentication
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../concepts/sharing/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Sharing
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../intro/showcase/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Showcase
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../intro/Governance/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Governance
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../intro/terms/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Terms
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../benchmark/" class="md-nav__link ">
+              
+  
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M0 256a256 256 0 1 1 512 0 256 256 0 1 1-512 0M288 96a32 32 0 1 0-64 0 32 32 0 1 0 64 0m-32 320c35.3 0 64-28.7 64-64 0-17.4-6.9-33.1-18.1-44.6L366 161.7c5.3-12.1-.2-26.3-12.3-31.6s-26.3.2-31.6 12.3L257.9 288H256c-35.3 0-64 28.7-64 64s28.7 64 64 64m-80-272a32 32 0 1 0-64 0 32 32 0 1 0 64 0M96 288a32 32 0 1 0 0-64 32 32 0 1 0 0 64m352-32a32 32 0 1 0-64 0 32 32 0 1 0 64 0"/></svg>
+  
+  <span class="md-ellipsis">
+    Benchmarking
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_2" id="__nav_2_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2">
+            <span class="md-nav__icon md-icon"></span>
+            Benchmarking
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_2" >
+        
+          
+          <label class="md-nav__link" for="__nav_2_2" id="__nav_2_2_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    AutoML Benchmark
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_2">
+            <span class="md-nav__icon md-icon"></span>
+            AutoML Benchmark
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../benchmark/automl/AutoML-Benchmark/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    AutoML Benchmark
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../benchmark/automl/important_params/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Important Parameters
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../benchmark/automl/benchmark_on_openml/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Benchmark on OpenML
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_2_4" >
+        
+          
+          <label class="md-nav__link" for="__nav_2_2_4" id="__nav_2_2_4_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Tutorials
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_2_2_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_2_2_4">
+            <span class="md-nav__icon md-icon"></span>
+            Tutorials
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../benchmark/automl/basic_example/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Basic Example - Random Forest
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../benchmark/automl/specific_task_fold_example/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Specific Task and Fold
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../ecosystem/" class="md-nav__link ">
+              
+  
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M512 32c0 113.6-84.6 207.5-194.2 222-7.1-53.4-30.6-101.6-65.3-139.3C290.8 46.3 364 0 448 0h32c17.7 0 32 14.3 32 32M0 96c0-17.7 14.3-32 32-32h32c123.7 0 224 100.3 224 224v192c0 17.7-14.3 32-32 32s-32-14.3-32-32V320C100.3 320 0 219.7 0 96"/></svg>
+  
+  <span class="md-ellipsis">
+    Ecosystem
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3" id="__nav_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3">
+            <span class="md-nav__icon md-icon"></span>
+            Ecosystem
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/Python_extensions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Python Integrations
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../ecosystem/Scikit-learn/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Scikit-learn
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_3_3" id="__nav_3_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_3">
+            <span class="md-nav__icon md-icon"></span>
+            Scikit-learn
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_3_2" >
+        
+          
+          <label class="md-nav__link" for="__nav_3_3_2" id="__nav_3_3_2_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Tutorials
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_3_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_3_2">
+            <span class="md-nav__icon md-icon"></span>
+            Tutorials
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/Scikit-learn/basic_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Basic tutorial
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/Scikit-learn/datasets_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Datasets
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/mlr/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    MLR
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/Weka/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Weka
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/MOA/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    MOA
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/Java/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Java
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../ecosystem/Rest/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    REST API
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../contributing/" class="md-nav__link ">
+              
+  
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M256 64a64 64 0 1 0-128 0 64 64 0 1 0 128 0M152.9 169.3c-23.7-8.4-44.5-24.3-58.8-45.8L74.6 94.2C64.8 79.5 45 75.6 30.2 85.4s-18.7 29.7-8.9 44.4L40.9 159c18.1 27.1 42.8 48.4 71.1 62.4V480c0 17.7 14.3 32 32 32s32-14.3 32-32v-96h32v96c0 17.7 14.3 32 32 32s32-14.3 32-32V221.6c29.1-14.2 54.4-36.2 72.7-64.2l18.2-27.9c9.6-14.8 5.4-34.6-9.4-44.3s-34.6-5.5-44.3 9.4L291 122.4c-21.8 33.4-58.9 53.6-98.8 53.6-12.6 0-24.9-2-36.6-5.8-.9-.3-1.8-.7-2.7-.9"/></svg>
+  
+  <span class="md-ellipsis">
+    Contributing
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_4" id="__nav_4_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4">
+            <span class="md-nav__icon md-icon"></span>
+            Contributing
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/OpenML-Docs/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Documentation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../services/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Backend
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_4_3" id="__nav_4_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4_3">
+            <span class="md-nav__icon md-icon"></span>
+            Backend
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/backend/Local-Installation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Local Installation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/backend/Datasets/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Datasets
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/backend/API-development/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Code structure
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/backend/Java-App/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Evaluation Engine
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_4" >
+        
+          
+          <label class="md-nav__link" for="__nav_4_4" id="__nav_4_4_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Frontend
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_4_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4_4">
+            <span class="md-nav__icon md-icon"></span>
+            Frontend
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/website/Website/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Getting started
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/website/Flask/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Flask backend
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/website/React/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    React frontend
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/website/Dash/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Dash visualizations
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_5" >
+        
+          
+          <label class="md-nav__link" for="__nav_4_5" id="__nav_4_5_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Clients
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_4_5">
+            <span class="md-nav__icon md-icon"></span>
+            Clients
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/clients/Client-API-Standards/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Client development
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/clients/metadata_definition/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Metadata definition
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/clients/Rest/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    REST API
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/Style/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Style guide
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../contributing/resources/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Resources
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../help/" class="md-nav__link">
+        
+  
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m15.07 11.25-.9.92C13.45 12.89 13 13.5 13 15h-2v-.5c0-1.11.45-2.11 1.17-2.83l1.24-1.26c.37-.36.59-.86.59-1.41a2 2 0 0 0-2-2 2 2 0 0 0-2 2H8a4 4 0 0 1 4-4 4 4 0 0 1 4 4 3.2 3.2 0 0 1-.93 2.25M13 19h-2v-2h2M12 2A10 10 0 0 0 2 12a10 10 0 0 0 10 10 10 10 0 0 0 10-10c0-5.53-4.5-10-10-10"/></svg>
+  
+  <span class="md-ellipsis">
+    Questions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../python/" class="md-nav__link ">
+              
+  
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9 3v15h3V3zm3 2 4 13 3-1-4-13zM5 5v13h3V5zM3 19v2h18v-2z"/></svg>
+  
+  <span class="md-ellipsis">
+    Python
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6" id="__nav_6_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6">
+            <span class="md-nav__icon md-icon"></span>
+            Python
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+          
+        
+      
+        
+          
+        
+      
+        
+      
+        
+          
+        
+      
+        
+          
+        
+      
+        
+          
+        
+      
+        
+          
+        
+      
+        
+          
+        
+      
+        
+          
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/datasets/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Code Reference
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1" id="__nav_6_1_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1">
+            <span class="md-nav__icon md-icon"></span>
+            Code Reference
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/base/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    base
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/cli/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    cli
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/config/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    config
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_5" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/evaluations/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    evaluations
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_5" id="__nav_6_1_5_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_5">
+            <span class="md-nav__icon md-icon"></span>
+            evaluations
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/evaluations/evaluation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    evaluation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/evaluations/functions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    functions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/exceptions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    exceptions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+          
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_7" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/extensions/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    extensions
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_7" id="__nav_6_1_7_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_7_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_7">
+            <span class="md-nav__icon md-icon"></span>
+            extensions
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/extensions/extension_interface/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    extension_interface
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/extensions/functions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    functions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_7_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/extensions/sklearn/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    sklearn
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_7_3" id="__nav_6_1_7_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="4" aria-labelledby="__nav_6_1_7_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_7_3">
+            <span class="md-nav__icon md-icon"></span>
+            sklearn
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/extensions/sklearn/extension/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    extension
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_8" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/flows/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    flows
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_8" id="__nav_6_1_8_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_8_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_8">
+            <span class="md-nav__icon md-icon"></span>
+            flows
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/flows/flow/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    flow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/flows/functions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    functions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_9" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/runs/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    runs
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_9" id="__nav_6_1_9_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_9">
+            <span class="md-nav__icon md-icon"></span>
+            runs
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/runs/functions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    functions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/runs/run/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    run
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/runs/trace/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    trace
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_10" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/setups/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    setups
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_10" id="__nav_6_1_10_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_10_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_10">
+            <span class="md-nav__icon md-icon"></span>
+            setups
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/setups/functions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    functions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/setups/setup/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    setup
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_11" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/study/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    study
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_11" id="__nav_6_1_11_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_11">
+            <span class="md-nav__icon md-icon"></span>
+            study
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/study/functions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    functions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/study/study/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    study
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_12" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../reference/tasks/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    tasks
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_6_1_12" id="__nav_6_1_12_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_12_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_1_12">
+            <span class="md-nav__icon md-icon"></span>
+            tasks
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/tasks/functions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    functions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/tasks/split/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    split
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/tasks/task/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    task
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/testing/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    testing
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../reference/utils/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    utils
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2" >
+        
+          
+          <label class="md-nav__link" for="__nav_6_2" id="__nav_6_2_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Examples
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2">
+            <span class="md-nav__icon md-icon"></span>
+            Examples
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_1" >
+        
+          
+          <label class="md-nav__link" for="__nav_6_2_1" id="__nav_6_2_1_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Basic
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_1_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_1">
+            <span class="md-nav__icon md-icon"></span>
+            Basic
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/20_basic/introduction_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Introduction tutorial &amp; Setup
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/20_basic/simple_datasets_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Datasets
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/20_basic/simple_flows_and_runs_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Flows and Runs
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/20_basic/simple_suites_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Benchmark suites
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_2" >
+        
+          
+          <label class="md-nav__link" for="__nav_6_2_2" id="__nav_6_2_2_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Extended
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_2">
+            <span class="md-nav__icon md-icon"></span>
+            Extended
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/configure_logging/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Logging
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/create_upload_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Dataset upload tutorial
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/custom_flow_/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Creating and Using a Custom Flow
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/datasets_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Datasets
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/fetch_evaluations_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Fetching Evaluations
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/fetch_runtimes_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Preparing tasks and scikit-learn models
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/flow_id_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Obtaining Flow IDs
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/flows_and_runs_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Flows and Runs
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/plot_svm_hyperparameters_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Plotting hyperparameter surfaces
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/run_setup_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Run Setup
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/study_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Benchmark studies
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/suites_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Benchmark suites
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/task_manual_iteration_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Tasks: retrieving splits
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/30_extended/tasks_tutorial/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Tasks
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_2_3" >
+        
+          
+          <label class="md-nav__link" for="__nav_6_2_3" id="__nav_6_2_3_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Paper
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_2_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_6_2_3">
+            <span class="md-nav__icon md-icon"></span>
+            Paper
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/40_paper/2015_neurips_feurer_example/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Feurer et al. (2015)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/40_paper/2018_ida_strang_example/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Strang et al. (2018)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/40_paper/2018_kdd_rijn_example/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    van Rijn and Hutter (2018)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../examples/40_paper/2018_neurips_perrone_example/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Perrone et al. (2018)
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../python/usage/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Usage
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../python/contributing/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Contributing
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../python/extensions/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Extensions
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../python/progress/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Changelog
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../pytorch/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    PyTorch
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_7" id="__nav_7_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_7">
+            <span class="md-nav__icon md-icon"></span>
+            PyTorch
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7_2" >
+        
+          
+          <label class="md-nav__link" for="__nav_7_2" id="__nav_7_2_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    API Reference
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_7_2_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_7_2">
+            <span class="md-nav__icon md-icon"></span>
+            API Reference
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/API%20reference/Callbacks/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Callbacks
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/API%20reference/Custom%20Datasets/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Custom Datasets
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/API%20reference/Metrics/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Metrics
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/API%20reference/OpenML%20Connection/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    OpenML Connection
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/API%20reference/Trainer/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Trainer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7_3" >
+        
+          
+          <label class="md-nav__link" for="__nav_7_3" id="__nav_7_3_label" tabindex="0">
+            
+  
+  <span class="md-ellipsis">
+    Examples
+  </span>
+  
+
+            <span class="md-nav__icon md-icon"></span>
+          </label>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_7_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_7_3">
+            <span class="md-nav__icon md-icon"></span>
+            Examples
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/Examples/Create%20Dataset%20and%20Task/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Create dataset and task - tiniest imagenet
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/Examples/Image%20Classification%20Task/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Image classification task
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/Examples/Pretrained%20Transformer%20Image%20Classification%20Task/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Pretrained Image classification example - Transformer
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/Examples/Sequential%20Classification%20Task/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Sequential classification
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/Examples/Tabular%20Classification/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Tabular classification
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/Limitations%20of%20the%20API/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Limitations of the API
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../pytorch/Philosophy%20behind%20the%20API%20Design/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Philosphy behind the API Design
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+    
+  
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        
+  
+  <span class="md-ellipsis">
+    TensorFlow
+  </span>
+  
+
+      </a>
+      
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#installation-instructions" class="md-nav__link">
+    <span class="md-ellipsis">
+      Installation Instructions:
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#usage" class="md-nav__link">
+    <span class="md-ellipsis">
+      Usage
+    </span>
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../julia/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Julia
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_9" id="__nav_9_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_9">
+            <span class="md-nav__icon md-icon"></span>
+            Julia
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../r/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    R
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+      
+        
+          
+        
+      
+        
+      
+        
+          
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../apiv2/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    APIv2
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_11" id="__nav_11_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11">
+            <span class="md-nav__icon md-icon"></span>
+            APIv2
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../apiv2/installation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Getting Started
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
+    
+      
+        
+      
+        
+      
+        
+      
+        
+      
+    
+    
+      
+      
+    
+    
+    <li class="md-nav__item md-nav__item--nested">
+      
+        
+        
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_11_3" >
+        
+          
+          
+          <div class="md-nav__link md-nav__container">
+            <a href="../apiv2/contributing/" class="md-nav__link ">
+              
+  
+  <span class="md-ellipsis">
+    Contributing
+  </span>
+  
+
+            </a>
+            
+              
+              <label class="md-nav__link " for="__nav_11_3" id="__nav_11_3_label" tabindex="0">
+                <span class="md-nav__icon md-icon"></span>
+              </label>
+            
+          </div>
+        
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_11_3_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_11_3">
+            <span class="md-nav__icon md-icon"></span>
+            Contributing
+          </label>
+          <ul class="md-nav__list" data-md-scrollfix>
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../apiv2/contributing/contributing/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Development
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../apiv2/contributing/tests/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Tests
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../apiv2/contributing/documentation/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Documentation
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../apiv2/contributing/project_overview/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Project Overview
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../apiv2/migration/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Changes
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+          </ul>
+        </nav>
+      
+    </li>
+  
+
+    
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+            
+              
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+                    
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#installation-instructions" class="md-nav__link">
+    <span class="md-ellipsis">
+      Installation Instructions:
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#usage" class="md-nav__link">
+    <span class="md-ellipsis">
+      Usage
+    </span>
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+                  </div>
+                </div>
+              </div>
+            
+          
+          
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+                
+                  
+
+  
+    <a href="https://github.com/openml/openml-tensorflow/edit/master/README.md" title="Edit this page" class="md-content__button md-icon">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20.71 7.04c.39-.39.39-1.04 0-1.41l-2.34-2.34c-.37-.39-1.02-.39-1.41 0l-1.84 1.83 3.75 3.75M3 17.25V21h3.75L17.81 9.93l-3.75-3.75z"/></svg>
+    </a>
+  
+  
+    
+      
+    
+    <a href="https://github.com/openml/openml-tensorflow/raw/master/README.md" title="View source of this page" class="md-content__button md-icon">
+      
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3 3 3 0 0 0 3-3 3 3 0 0 0-3-3m0 8a5 5 0 0 1-5-5 5 5 0 0 1 5-5 5 5 0 0 1 5 5 5 5 0 0 1-5 5m0-12.5C7 4.5 2.73 7.61 1 12c1.73 4.39 6 7.5 11 7.5s9.27-3.11 11-7.5c-1.73-4.39-6-7.5-11-7.5"/></svg>
+    </a>
+  
+
+
+<h1 id="tensorflow-extension-for-openml-python">Tensorflow extension for OpenML python<a class="headerlink" href="#tensorflow-extension-for-openml-python" title="Permanent link">&para;</a></h1>
+<p>Tensorflow extension for <a href="https://github.com/openml/openml-python">openml-python API</a>.</p>
+<h4 id="installation-instructions">Installation Instructions:<a class="headerlink" href="#installation-instructions" title="Permanent link">&para;</a></h4>
+<p><code>pip install openml-tensorflow</code></p>
+<p>PyPi link <a href="https://pypi.org/project/openml-keras/">https://pypi.org/project/openml-keras/</a></p>
+<h4 id="usage">Usage<a class="headerlink" href="#usage" title="Permanent link">&para;</a></h4>
+<p>Import openML libraries
+<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1">1</a></span>
+<span class="normal"><a href="#__codelineno-0-2">2</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1"></a><span class="kn">import</span> <span class="nn">openml</span>
+<a id="__codelineno-0-2" name="__codelineno-0-2"></a><span class="kn">import</span> <span class="nn">openml_tensorflow</span>
+</code></pre></div></td></tr></table></div>
+Create  and compile a keras model
+<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-1-1"> 1</a></span>
+<span class="normal"><a href="#__codelineno-1-2"> 2</a></span>
+<span class="normal"><a href="#__codelineno-1-3"> 3</a></span>
+<span class="normal"><a href="#__codelineno-1-4"> 4</a></span>
+<span class="normal"><a href="#__codelineno-1-5"> 5</a></span>
+<span class="normal"><a href="#__codelineno-1-6"> 6</a></span>
+<span class="normal"><a href="#__codelineno-1-7"> 7</a></span>
+<span class="normal"><a href="#__codelineno-1-8"> 8</a></span>
+<span class="normal"><a href="#__codelineno-1-9"> 9</a></span>
+<span class="normal"><a href="#__codelineno-1-10">10</a></span>
+<span class="normal"><a href="#__codelineno-1-11">11</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1"></a><span class="n">model</span> <span class="o">=</span> <span class="n">tensorflow</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">models</span><span class="o">.</span><span class="n">Sequential</span><span class="p">([</span>
+<a id="__codelineno-1-2" name="__codelineno-1-2"></a>    <span class="n">tensorflow</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">BatchNormalization</span><span class="p">(),</span>
+<a id="__codelineno-1-3" name="__codelineno-1-3"></a>    <span class="n">tensorflow</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">units</span><span class="o">=</span><span class="mi">1024</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="n">tensorflow</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">activations</span><span class="o">.</span><span class="n">relu</span><span class="p">),</span>
+<a id="__codelineno-1-4" name="__codelineno-1-4"></a>    <span class="n">tensorflow</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">rate</span><span class="o">=</span><span class="mf">0.4</span><span class="p">),</span>
+<a id="__codelineno-1-5" name="__codelineno-1-5"></a>    <span class="n">tensorflow</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">units</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="n">tensorflow</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">activations</span><span class="o">.</span><span class="n">softmax</span><span class="p">),</span>
+<a id="__codelineno-1-6" name="__codelineno-1-6"></a><span class="p">])</span>
+<a id="__codelineno-1-7" name="__codelineno-1-7"></a>
+<a id="__codelineno-1-8" name="__codelineno-1-8"></a><span class="c1"># We will compile using the Adam optimizer while targeting accuracy.</span>
+<a id="__codelineno-1-9" name="__codelineno-1-9"></a><span class="n">model</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">optimizer</span><span class="o">=</span><span class="s1">&#39;adam&#39;</span><span class="p">,</span>
+<a id="__codelineno-1-10" name="__codelineno-1-10"></a>              <span class="n">loss</span><span class="o">=</span><span class="s1">&#39;sparse_categorical_crossentropy&#39;</span><span class="p">,</span>
+<a id="__codelineno-1-11" name="__codelineno-1-11"></a>              <span class="n">metrics</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;accuracy&#39;</span><span class="p">])</span>
+</code></pre></div></td></tr></table></div>
+Download the task from openML and run the model on task.
+<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-2-1">1</a></span>
+<span class="normal"><a href="#__codelineno-2-2">2</a></span>
+<span class="normal"><a href="#__codelineno-2-3">3</a></span>
+<span class="normal"><a href="#__codelineno-2-4">4</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1"></a><span class="n">task</span> <span class="o">=</span> <span class="n">openml</span><span class="o">.</span><span class="n">tasks</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="mi">3573</span><span class="p">)</span>
+<a id="__codelineno-2-2" name="__codelineno-2-2"></a><span class="n">run</span> <span class="o">=</span> <span class="n">openml</span><span class="o">.</span><span class="n">runs</span><span class="o">.</span><span class="n">run_model_on_task</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="n">avoid_duplicate_runs</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+<a id="__codelineno-2-3" name="__codelineno-2-3"></a><span class="n">run</span><span class="o">.</span><span class="n">publish</span><span class="p">()</span>
+<a id="__codelineno-2-4" name="__codelineno-2-4"></a><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;URL for run: </span><span class="si">%s</span><span class="s1">/run/</span><span class="si">%d</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">openml</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">server</span><span class="p">,</span> <span class="n">run</span><span class="o">.</span><span class="n">run_id</span><span class="p">))</span>
+</code></pre></div></td></tr></table></div></p>
+<p>This library is currently under development, please report any bugs or feature reuest in issues section.</p>
+
+
+
+
+
+
+
+  
+    
+  
+  
+    
+  
+
+
+  <aside class="md-source-file">
+    
+      
+  <span class="md-source-file__fact">
+    <span class="md-icon" title="Last update">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
+    </span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
+  </span>
+
+    
+    
+      
+  <span class="md-source-file__fact">
+    <span class="md-icon" title="Created">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3z"/></svg>
+    </span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 31, 2024</span>
+  </span>
+
+    
+    
+    
+  </aside>
+
+
+
+
+
+                
+              </article>
+            </div>
+          
+          
+  <script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var labels=set.querySelector(".tabbed-labels");for(var tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
+
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+        
+      </main>
+      
+        <footer class="md-footer">
+  
+    
+      
+      <nav class="md-footer__inner md-grid" aria-label="Footer" >
+        
+          
+          <a href="../pytorch/Philosophy%20behind%20the%20API%20Design/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Philosphy behind the API Design">
+            <div class="md-footer__button md-icon">
+              
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+            </div>
+            <div class="md-footer__title">
+              <span class="md-footer__direction">
+                Previous
+              </span>
+              <div class="md-ellipsis">
+                Philosphy behind the API Design
+              </div>
+            </div>
+          </a>
+        
+        
+          
+          <a href="../julia/" class="md-footer__link md-footer__link--next" aria-label="Next: Home">
+            <div class="md-footer__title">
+              <span class="md-footer__direction">
+                Next
+              </span>
+              <div class="md-ellipsis">
+                Home
+              </div>
+            </div>
+            <div class="md-footer__button md-icon">
+              
+              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
+            </div>
+          </a>
+        
+      </nav>
+    
+  
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+  
+  
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+  
+</div>
+      
+    </div>
+  </div>
+</footer>
+      
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+    
+    
+    <script id="__config" type="application/json">{"base": "..", "features": ["content.code.copy", "content.action.edit", "content.action.view", "content.tabs.link", "navigation.indexes", "navigation.tabs", "navigation.tabs.sticky", "navigation.footer", "search.highlight", "search.suggest", "toc.follow"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
+    
+    
+      <script src="../assets/javascripts/bundle.525ec568.min.js"></script>
+      
+        <script src="../js/extra.js"></script>
+      
+    
+  </body>
+</html>
\ No newline at end of file