Switch presets to final Kaggle location (#2216)

* Convert initial kaggle handles * Add Vision transformer kaggle switch * Add kagglehub requirement and auth, fix formatting * Special case input shape arg * Add kaggle username and key to CI * Add kaggle authentication to Keras 2 actions * Add test line * Add an extra space * Fix input shape build on load_preset * Fix format issues
keras-team · Dec 11, 2023 · cc084f8 · cc084f8
1 parent d30f78f
commit cc084f8
Show file tree

Hide file tree

Showing 22 changed files with 102 additions and 84 deletions.
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
@@ -41,6 +41,8 @@ jobs:
         pip install -e ".[tests]" --progress-bar off --upgrade
     - name: Test with pytest
       env:
+        KAGGLE_USERNAME:  ${{ secrets.KAGGLE_USERNAME }}
+        KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
         TEST_CUSTOM_OPS: false
       run: |
         pytest keras_cv/ --ignore keras_cv/models/legacy/ --durations 0
@@ -77,6 +79,8 @@ jobs:
       env:
         TEST_CUSTOM_OPS: false # TODO(ianstenbit): test custom ops, or figure out what our story is here
         KERAS_BACKEND: ${{ matrix.backend }}
+        KAGGLE_USERNAME:  ${{ secrets.KAGGLE_USERNAME}}
+        KAGGLE_KEY: ${{ secrets.KAGGLE_KEY}}
         JAX_ENABLE_X64: true
       run: |
         pytest keras_cv/bounding_box \

diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml
@@ -50,6 +50,7 @@ steps:
     '--ext-str', 'backend=$_BACKEND',
     '-o', 'output.yaml',
   ]
+  secretEnv: ['KAGGLE_USERNAME', 'KAGGLE_KEY']
 - name: 'gcr.io/cloud-builders/gcloud'
   id: create-job
   waitFor:
@@ -75,3 +76,9 @@ options:
   volumes:
   - name: go-modules
     path: /go
+availableSecrets:
+  secretManager:
+  - versionName: projects/$PROJECT_ID/secrets/KAGGLE_USERNAME/versions/latest
+    env: 'KAGGLE_USERNAME'
+  - versionName: projects/$PROJECT_ID/secrets/KAGGLE_KEY/versions/latest
+    env: 'KAGGLE_KEY'
diff --git a/keras_cv/models/backbones/csp_darknet/csp_darknet_backbone_presets.py b/keras_cv/models/backbones/csp_darknet/csp_darknet_backbone_presets.py
@@ -26,7 +26,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/csp_darknet_tiny",
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_tiny/1",
     },
     "csp_darknet_s": {
         "metadata": {
@@ -39,7 +39,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/csp_darknet_s",
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_s/1",
     },
     "csp_darknet_m": {
         "metadata": {
@@ -52,7 +52,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/csp_darknet_m",
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_m/1",
     },
     "csp_darknet_l": {
         "metadata": {
@@ -65,7 +65,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/csp_darknet_l",
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_l/1",
     },
     "csp_darknet_xl": {
         "metadata": {
@@ -78,7 +78,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/csp_darknet_xl",
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_xl/1",
     },
 }
 
@@ -95,7 +95,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/csp_darknet_tiny_imagenet",
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_tiny_imagenet/1",  # noqa: E501
     },
     "csp_darknet_l_imagenet": {
         "metadata": {
@@ -109,7 +109,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/csp_darknet_l_imagenet",
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_l_imagenet/1",
     },
 }
 

diff --git a/keras_cv/models/backbones/csp_darknet/csp_darknet_backbone_test.py b/keras_cv/models/backbones/csp_darknet/csp_darknet_backbone_test.py
@@ -113,6 +113,9 @@ def test_saved_alias_model(self):
         )
 
     def test_feature_pyramid_inputs(self):
+        import os
+
+        print("HERE", os.environ["KAGGLE_USERNAME"])
         model = CSPDarkNetLBackbone()
         backbone_model = get_feature_extractor(
             model,

diff --git a/keras_cv/models/backbones/densenet/densenet_backbone_presets.py b/keras_cv/models/backbones/densenet/densenet_backbone_presets.py
@@ -18,19 +18,19 @@
         "metadata": {
             "description": "DenseNet model with 121 layers.",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/densenet121",
+        "kaggle_handle": "kaggle://keras/densenet/densenet121/1",
     },
     "densenet169": {
         "metadata": {
             "description": "DenseNet model with 169 layers.",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/densenet169",
+        "kaggle_handle": "kaggle://keras/densenet/densenet169/1",
     },
     "densenet201": {
         "metadata": {
             "description": "DenseNet model with 201 layers.",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/densenet201",
+        "kaggle_handle": "kaggle://keras/densenet/densenet201/1",
     },
 }
 
@@ -42,7 +42,7 @@
                 "classification task."
             ),
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/densenet121_imagenet",
+        "kaggle_handle": "kaggle://keras/densenet/densenet121_imagenet/1",
     },
     "densenet169_imagenet": {
         "metadata": {
@@ -51,7 +51,7 @@
                 "classification task."
             ),
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/densenet169_imagenet",
+        "kaggle_handle": "kaggle://keras/densenet/densenet169_imagenet/1",
     },
     "densenet201_imagenet": {
         "metadata": {
@@ -60,7 +60,7 @@
                 "classification task."
             ),
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/densenet201_imagenet",
+        "kaggle_handle": "kaggle://keras/densenet/densenet201_imagenet/1",
     },
 }
 

diff --git a/keras_cv/models/backbones/efficientnet_v2/efficientnet_v2_backbone_presets.py b/keras_cv/models/backbones/efficientnet_v2/efficientnet_v2_backbone_presets.py
@@ -24,7 +24,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_s",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_s/1",
     },
     "efficientnetv2_m": {
         "metadata": {
@@ -35,7 +35,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_m",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_m/1",
     },
     "efficientnetv2_l": {
         "metadata": {
@@ -47,7 +47,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_l",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_l/1",
     },
     "efficientnetv2_b0": {
         "metadata": {
@@ -60,7 +60,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_b0",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_b0/1",
     },
     "efficientnetv2_b1": {
         "metadata": {
@@ -73,7 +73,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_b1",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_b1/1",
     },
     "efficientnetv2_b2": {
         "metadata": {
@@ -86,7 +86,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_b2",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_b2/1",
     },
     "efficientnetv2_b3": {
         "metadata": {
@@ -99,7 +99,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_b3",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_b3/1",
     },
 }
 
@@ -117,7 +117,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_s_imagenet",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_s_imagenet/1",  # noqa: E501
     },
     "efficientnetv2_b0_imagenet": {
         "metadata": {
@@ -134,7 +134,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_b0_imagenet",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_b0_imagenet/1",  # noqa: E501
     },
     "efficientnetv2_b1_imagenet": {
         "metadata": {
@@ -151,7 +151,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_b1_imagenet",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_b1_imagenet/1",  # noqa: E501
     },
     "efficientnetv2_b2_imagenet": {
         "metadata": {
@@ -168,7 +168,7 @@
             "official_name": "EfficientNetV2",
             "path": "efficientnetv2",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/efficientnetv2_b2_imagenet",
+        "kaggle_handle": "kaggle://keras/efficientnetv2/efficientnetv2_b2_imagenet/1",  # noqa: E501
     },
 }
 

diff --git a/keras_cv/models/backbones/mix_transformer/mix_transformer_backbone_presets.py b/keras_cv/models/backbones/mix_transformer/mix_transformer_backbone_presets.py
@@ -23,7 +23,7 @@
             "official_name": "MiT",
             "path": "mit",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mit_b0",
+        "kaggle_handle": "kaggle://keras/mit/mit_b0/1",
     },
     "mit_b1": {
         "metadata": {
@@ -34,7 +34,7 @@
             "official_name": "MiT",
             "path": "mit",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mit_b1",
+        "kaggle_handle": "kaggle://keras/mit/mit_b1/1",
     },
     "mit_b2": {
         "metadata": {
@@ -45,7 +45,7 @@
             "official_name": "MiT",
             "path": "mit",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mit_b2",
+        "kaggle_handle": "kaggle://keras/mit/mit_b2/1",
     },
     "mit_b3": {
         "metadata": {
@@ -56,7 +56,7 @@
             "official_name": "MiT",
             "path": "mit",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mit_b3",
+        "kaggle_handle": "kaggle://keras/mit/mit_b3/1",
     },
     "mit_b4": {
         "metadata": {
@@ -67,7 +67,7 @@
             "official_name": "MiT",
             "path": "mit",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mit_b4",
+        "kaggle_handle": "kaggle://keras/mit/mit_b4/1",
     },
     "mit_b5": {
         "metadata": {
@@ -78,7 +78,7 @@
             "official_name": "MiT",
             "path": "mit",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mit_b5",
+        "kaggle_handle": "kaggle://keras/mit/mit_b5/1",
     },
 }
 
@@ -92,7 +92,7 @@
             "official_name": "MiT",
             "path": "mit",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mit_b0_imagenet",
+        "kaggle_handle": "kaggle://keras/mit/mit_b0_imagenet/1",
     },
 }
 

diff --git a/keras_cv/models/backbones/mobilenet_v3/mobilenet_v3_backbone_presets.py b/keras_cv/models/backbones/mobilenet_v3/mobilenet_v3_backbone_presets.py
@@ -25,7 +25,7 @@
             "official_name": "MobileNetV3",
             "path": "mobilenetv3",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mobilenet_v3_small",
+        "kaggle_handle": "kaggle://keras/mobilenetv3/mobilenet_v3_small/1",
     },
     "mobilenet_v3_large": {
         "metadata": {
@@ -38,7 +38,7 @@
             "official_name": "MobileNetV3",
             "path": "mobilenetv3",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mobilenet_v3_large",
+        "kaggle_handle": "kaggle://keras/mobilenetv3/mobilenet_v3_large/1",
     },
 }
 
@@ -55,7 +55,7 @@
             "official_name": "MobileNetV3",
             "path": "mobilenetv3",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mobilenet_v3_large_imagenet",
+        "kaggle_handle": "kaggle://keras/mobilenetv3/mobilenet_v3_large_imagenet/1",  # noqa: E501
     },
     "mobilenet_v3_small_imagenet": {
         "metadata": {
@@ -69,7 +69,7 @@
             "official_name": "MobileNetV3",
             "path": "mobilenetv3",
         },
-        "kaggle_handle": "gs://keras-cv-kaggle/mobilenet_v3_small_imagenet",
+        "kaggle_handle": "kaggle://keras/mobilenetv3/mobilenet_v3_small_imagenet/1",  # noqa: E501
     },
 }