Merge pull request #49 from allenai/favyen/marine-infra

Add marine infrastructure dataset and model config

Author: favyen2

.github/workflows/build_test.yaml

@@ -109,18 +109,27 @@ jobs:
       - name: Run unit tests with Docker Compose
         run: |
           docker compose -f docker-compose.yaml run \
+            -e CI="true" \
             test pytest tests/unit/
 
       - name: Run tests with Docker Compose
         run: |
           docker compose -f docker-compose.yaml run \
+            -e CI="true" \
             -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
             -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
             -v ${{env.GOOGLE_GHA_CREDS_PATH}}:/tmp/gcp-credentials.json:ro \
             -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json \
             -e RSLP_BUCKET=rslearn-eai \
             -e RSLP_PREFIX=gs://rslearn-eai \
-            test pytest tests/integration/ --ignore tests/integration_slow/ -vv
+            -e BKT_PROJECT_ID=earthsystem-dev-c3po \
+            -e BKT_BUCKET_NAME=rslp-tests \
+            -e BKT_BIGTABLE_PROJECT_ID=earthsystem-dev-c3po \
+            -e BKT_BIGTABLE_INSTANCE_ID=rslp-bigtable-test-instance \
+            -e TEST_PUBSUB_PROJECT=earthsystem-dev-c3po \
+            -e TEST_PUBSUB_TOPIC=rslearn_projects_test_topic \
+            -e TEST_PUBSUB_SUBSCRIPTION=rslearn_projects_test_subscription \
+            test pytest tests/integration/ -vv
 
 
       - name: Clean up

Dockerfile

@@ -1,7 +1,7 @@
 FROM pytorch/pytorch:2.5.0-cuda11.8-cudnn9-runtime@sha256:d15e9803095e462e351f097fb1f5e7cdaa4f5e855d7ff6d6f36ec4c2aa2938ea
 
 RUN apt update
-RUN apt install -y libpq-dev ffmpeg libsm6 libxext6 git
+RUN apt install -y libpq-dev ffmpeg libsm6 libxext6 git wget
 
 # Install rslearn.
 # We use git clone and then git checkout instead of git clone -b so that the user could
@@ -27,4 +27,12 @@ ENV PYTHONPATH="${PYTHONPATH}:."
 COPY . /opt/rslearn_projects/
 # install rslp package
 RUN pip install --no-cache-dir /opt/rslearn_projects
+
+# Build Satlas smooth_point_labels_viterbi.go program.
+WORKDIR /opt/rslearn_projects/rslp/satlas/scripts
+RUN wget https://go.dev/dl/go1.22.12.linux-amd64.tar.gz
+RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.12.linux-amd64.tar.gz
+ENV PATH="${PATH}:/usr/local/go/bin"
+RUN go build smooth_point_labels_viterbi.go
+
 WORKDIR /opt/rslearn_projects

ai2_docs/README.md

@@ -33,6 +33,10 @@ rslearn datasets, model checkpoints, etc. The easiest way is to create a `.env`
 
     RSLP_PREFIX=gs://rslearn-eai
     RSLP_WEKA_PREFIX=weka://dfive-default/rslearn-eai
+    BKT_PROJECT_ID=ai2-prior-satlas
+    BKT_BUCKET_NAME=satlas-explorer-data
+    BKT_BIGTABLE_PROJECT_ID=ai2-prior-satlas
+    BKT_BIGTABLE_INSTANCE_ID=satlas
 
 You will also need to setup GCP credentials that have access to this bucket.
 
@@ -42,19 +46,6 @@ launcher like `rslp.launch_beaker`, then it isn't needed since the credentials a
 already configured as secrets on the platform, but you would need to setup your Beaker
 or other platform credentials to be able to launch the jobs.
 
-TODO: update GCP/W&B to use service accounts.
-
-Currently, until https://github.com/allenai/rslearn/issues/33 is resolved, model config
-files use S3-compatable API to access GCS rather than GCS directly. Therefore, you need
-to set up environment variables to provide the appropriate credentials:
-
-    S3_ACCESS_KEY_ID=GOOG...
-    S3_SECRET_ACCESS_KEY=...
-
-You can create these credentials at
-https://console.cloud.google.com/storage/settings;tab=interoperability?hl=en&project=skylight-proto-1
-under "Access keys for your user account".
-
 
 Usage
 -----

data/satlas/marine_infra/config.json

@@ -0,0 +1,111 @@
+{
+  "layers": {
+    "label": {
+      "type": "vector"
+    },
+    "mask": {
+      "band_sets": [
+        {
+          "bands": [
+            "mask"
+          ],
+          "dtype": "uint8",
+          "format": {
+            "format": "png",
+            "name": "single_image"
+          }
+        }
+      ],
+      "type": "raster"
+    },
+    "output": {
+      "format": {
+        "coordinate_mode": "pixel",
+        "name": "geojson"
+      },
+      "type": "vector"
+    },
+    "sentinel2": {
+      "band_sets": [
+        {
+          "bands": [
+            "B02",
+            "B03",
+            "B04",
+            "B08"
+          ],
+          "dtype": "uint16",
+          "format": {
+            "geotiff_options": {
+              "compress": "zstd",
+              "predictor": 2,
+              "zstd_level": 1
+            },
+            "name": "geotiff"
+          }
+        },
+        {
+          "bands": [
+            "B05",
+            "B06",
+            "B07",
+            "B8A",
+            "B11",
+            "B12"
+          ],
+          "dtype": "uint16",
+          "format": {
+            "geotiff_options": {
+              "compress": "zstd",
+              "predictor": 2,
+              "zstd_level": 1
+            },
+            "name": "geotiff"
+          },
+          "zoom_offset": -1
+        },
+        {
+          "bands": [
+            "B01",
+            "B09",
+            "B10"
+          ],
+          "dtype": "uint16",
+          "format": {
+            "geotiff_options": {
+              "compress": "zstd",
+              "predictor": 2,
+              "zstd_level": 1
+            },
+            "name": "geotiff"
+          },
+          "zoom_offset": -2
+        }
+      ],
+      "data_source": {
+        "harmonize": true,
+        "index_cache_dir": "cache/sentinel2",
+        "max_cloud_cover": 50,
+        "max_time_delta": "0d",
+        "modality": "L1C",
+        "name": "rslp.satlas.data_sources.MonthlySentinel2",
+        "query_config": {
+          "max_matches": 4
+        },
+        "sort_by": "cloud_cover",
+        "use_rtree_index": false
+      },
+      "type": "raster"
+    }
+  },
+  "tile_store": {
+    "class_path": "rslearn.tile_stores.default.DefaultTileStore",
+    "init_args": {
+      "geotiff_options": {
+        "compress": "zstd",
+        "predictor": 2,
+        "zstd_level": 1
+      }
+    }
+  }
+}