Merge branch 'master' into feat/openfga-policy-engine

permitio · Nov 20, 2024 · 96a55db · 96a55db
2 parents b7f27e9 + fee14db
commit 96a55db
Show file tree

Hide file tree

Showing 24 changed files with 171 additions and 66 deletions.
diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml
@@ -86,13 +86,13 @@ jobs:
         run: sed 's/:latest/:test/g' docker/docker-compose-example.yml > docker/docker-compose-test.yml
 
       - name: Bring up stack
-        run: docker-compose -f docker/docker-compose-test.yml up -d
+        run: docker compose -f docker/docker-compose-test.yml up -d
 
       - name: Check if OPA is healthy
         run: ./scripts/wait-for.sh -t 60 http://localhost:8181/v1/data/users -- sleep 10 && curl -s "http://localhost:8181/v1/data/users" | jq '.result.bob.location.country == "US"'
 
       - name: Output container logs
-        run: docker-compose -f docker/docker-compose-test.yml logs
+        run: docker compose -f docker/docker-compose-test.yml logs
 
       # PUSH PHASE
       - name: Output local docker images

diff --git a/README.md b/README.md
@@ -100,7 +100,7 @@ Besides Permit, OPAL is being used in Production in **Tesla**, **Walmart**, **Th
 
 ```
 curl -L https://raw.githubusercontent.com/permitio/opal/master/docker/docker-compose-example.yml \
-> docker-compose.yml && docker-compose up
+> docker-compose.yml && docker compose up
 ```
 
 <p>

diff --git a/app-tests/run.sh b/app-tests/run.sh
@@ -158,4 +158,20 @@ function main {
   # TODO: Test statistics feature again after broadcaster restart (should first fix statistics bug)
 }
 
-main
+# Retry test in case of failure to avoid flakiness
+MAX_RETRIES=5
+RETRY_COUNT=0
+
+while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  echo "Running test (attempt $((RETRY_COUNT+1)) of $MAX_RETRIES)..."
+  main && break
+  RETRY_COUNT=$((RETRY_COUNT + 1))
+  echo "Test failed, retrying..."
+done
+
+if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
+  echo "Tests failed after $MAX_RETRIES attempts."
+  exit 1
+fi
+
+echo "Tests passed successfully."
diff --git a/cedar-agent b/cedar-agent
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -39,7 +39,7 @@ RUN chmod +x ./wait-for.sh
 RUN ln -s /opal/wait-for.sh /usr/wait-for.sh
 
 # netcat (nc) is used by the wait-for.sh script
-RUN apt-get update && apt-get install -y netcat-traditional jq && apt-get clean
+RUN apt-get update && apt-get install -y netcat-traditional jq wget && apt-get clean
 
 # copy startup script (create link at old path to maintain backward compatibility)
 COPY ./scripts/start.sh .
@@ -116,6 +116,7 @@ COPY --from=opa-extractor /opal/opa ./opa
 
 # enable inline OPA
 ENV OPAL_INLINE_OPA_ENABLED=true
+ENV OPAL_INLINE_OPA_EXEC_PATH=/opal/opa
 # expose opa port
 EXPOSE 8181
 USER opal
@@ -134,6 +135,7 @@ COPY --from=cedar-builder /tmp/cedar-agent/target/*/cedar-agent /bin/cedar-agent
 # enable inline Cedar agent
 ENV OPAL_POLICY_STORE_TYPE=CEDAR
 ENV OPAL_INLINE_CEDAR_ENABLED=true
+ENV OPAL_INLINE_CEDAR_EXEC_PATH=/bin/cedar-agent
 ENV OPAL_INLINE_CEDAR_CONFIG='{"addr": "0.0.0.0:8180"}'
 ENV OPAL_POLICY_STORE_URL=http://localhost:8180
 # expose cedar port

diff --git a/docker/docker-compose-api-policy-source-example.yml b/docker/docker-compose-api-policy-source-example.yml
@@ -1,3 +1,5 @@
+name: opal-api-policy-source-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

diff --git a/docker/docker-compose-example-cedar.yml b/docker/docker-compose-example-cedar.yml
@@ -1,3 +1,5 @@
+name: opal-cedar-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

diff --git a/docker/docker-compose-example.yml b/docker/docker-compose-example.yml
@@ -1,3 +1,5 @@
+name: opal-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

diff --git a/docker/docker-compose-git-webhook.yml b/docker/docker-compose-git-webhook.yml
@@ -1,3 +1,5 @@
+name: opal-git-webhook-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

diff --git a/docker/docker-compose-scopes-example.yml b/docker/docker-compose-scopes-example.yml
@@ -1,3 +1,5 @@
+name: opal-scopes-example
+
 services:
   redis:
     image: redis

diff --git a/docker/docker-compose-with-callbacks.yml b/docker/docker-compose-with-callbacks.yml
@@ -1,3 +1,5 @@
+name: opal-callbacks-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

diff --git a/docker/docker-compose-with-kafka-example.yml b/docker/docker-compose-with-kafka-example.yml
@@ -1,3 +1,4 @@
+name: opal-kafka-example
 
 services:
   # Based on: https://developer.confluent.io/quickstart/kafka-docker/

diff --git a/docker/docker-compose-with-oauth-initial.yml b/docker/docker-compose-with-oauth-initial.yml
@@ -1,3 +1,5 @@
+name: opal-oauth-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

diff --git a/docker/docker-compose-with-rate-limiting.yml b/docker/docker-compose-with-rate-limiting.yml
@@ -1,3 +1,5 @@
+name: opal-rate-limiting-example
+
 # This docker compose example shows how to configure OPAL's rate limiting feature
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use

diff --git a/docker/docker-compose-with-security.yml b/docker/docker-compose-with-security.yml
@@ -1,3 +1,5 @@
+name: opal-security-example
+
 # this docker compose file is relying on external environment variables!
 # run it by running the script: ./run-example-with-security.sh
 services:

diff --git a/docker/docker-compose-with-statistics.yml b/docker/docker-compose-with-statistics.yml
@@ -1,3 +1,5 @@
+name: opal-statistics-example
+
 services:
   # When scaling the opal-server to multiple nodes and/or multiple workers, we use
   # a *broadcast* channel to sync between all the instances of opal-server.

diff --git a/documentation/docs/getting-started/configuration.mdx b/documentation/docs/getting-started/configuration.mdx
@@ -40,7 +40,7 @@ Please use this table as a reference.
 | AUTH_PUBLIC_KEY                         |                                                                                                                                                                                 |         |
 | AUTH_JWT_ALGORITHM                      | JWT algorithm. See possible values [here](https://pyjwt.readthedocs.io/en/stable/algorithms.html).                                                                              |         |
 | AUTH_JWT_AUDIENCE                       |                                                                                                                                                                                 |         |
-| AUTH_JWT_ISSUER                         |
+| AUTH_JWT_ISSUER                         |                                                                                                                                                                                 |                           |
 
 ## OPAL Server Configuration Variables
 
@@ -121,7 +121,7 @@ Please use this table as a reference.
 ## OPAL Client Configuration Variables
 
 | Variables                             | Description                                                                                                                                                                                                                                                                    | Example                              |
-| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------ |
+| ------------------------------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------ |
 | POLICY_STORE_TYPE                     |                                                                                                                                                                                                                                                                                |                                      |
 | POLICY_STORE_AUTH_TYPE                | The authentication method for connecting to the policy store. Possible values are `oauth` or `token`                                                                                                                                                                           |                                      |
 | POLICY_STORE_AUTH_TOKEN               | The authentication (bearer) token OPAL client will use to authenticate against the policy store (i.e: OPA agent).                                                                                                                                                              |                                      |
@@ -131,8 +131,13 @@ Please use this table as a reference.
 | POLICY_STORE_CONN_RETRY               | Retry options when connecting to the policy store (i.e. the agent that handles the policy, e.g. OPA).                                                                                                                                                                          |                                      |
 | POLICY_STORE_POLICY_PATHS_TO_IGNORE   | Which policy paths pushed to the client should be ignored. List of glob style paths, or paths without wildcards but ending with "/\*\*" indicating a parent path (ignoring all under it).                                                                                      |                                      |
 | INLINE_OPA_ENABLED                    | Whether or not OPAL should run OPA by itself in the same container.                                                                                                                                                                                                            |                                      |
+| INLINE_OPA_EXEC_PATH                  | The path to the OPA executable.                                                                                                                                                                                                                                                |                                      |
 | INLINE_OPA_CONFIG                     | If inline OPA is indeed enabled, the user can set the [server configuration options](https://docs.opal.ac/getting-started/running-opal/run-opal-client/opa-runner-parameters) that affects how OPA will start when running `opa run --server` inline.  Watch escaping quotes.  | \{"config_file":"/mnt/opa/config"\}  |
 | INLINE_OPA_LOG_FORMAT                 |                                                                                                                                                                                                                                                                                |                                      |
+| INLINE_CEDAR_ENABLED                  | Whether or not OPAL should run Cedar agent by itself in the same container.                                                                                                                                                                                                    |                                      |
+| INLINE_CEDAR_EXEC_PATH                | The path to the Cedar agent executable.                                                                                                                                                                                                                                        |                                      |
+| INLINE_CEDAR_CONFIG                   | If inline Cedar is indeed enabled, provide options for running the Cedar agent                                                                                                                                                                                                 |                                      |
+| INLINE_CEDAR_LOG_FORMAT               |                                                                                                                                                                                                                                                                                |                                      |
 | KEEP_ALIVE_INTERVAL                   |                                                                                                                                                                                                                                                                                |                                      |
 | OFFLINE_MODE_ENABLED                  | If set, opal client will try to load policy store from backup file and operate even if server is unreachable. Ignored if INLINE_OPA_ENABLED=False                                                                                                                              |                                      |
 | STORE_BACKUP_PATH                     | Path to backup policy store's data to                                                                                                                                                                                                                                          |                                      |

diff --git a/documentation/docs/tutorials/run_opal_with_kafka.mdx b/documentation/docs/tutorials/run_opal_with_kafka.mdx
@@ -39,7 +39,7 @@ But (in version OPAL 0.1.21 and later) you can also use the `OPAL_BROADCAST_CHAN
   - a Kafka topic is used to control which servers share clients and events.
   - OPAL topics control which clients receive which policy or data events.
 
-## Docker-compose Example
+## Docker Compose Example
 
 Check out `docker/docker-compose-with-kafka-example.yml` for running docker compose with OPAL-server, OPAL-client, Zookeeper, and Kafka.
 

diff --git a/documentation/docs/tutorials/run_opal_with_pulsar.mdx b/documentation/docs/tutorials/run_opal_with_pulsar.mdx
@@ -71,7 +71,7 @@ volumes:
 Run this configuration with:
 
 ```bash
-docker-compose up --force-recreate
+docker compose up --force-recreate
 ```
 
 Allow a few seconds for Apache Pulsar and OPAL to start up before testing connectivity.
@@ -93,10 +93,10 @@ You should see the effect in:
 
 | Backend  | Environment Variable                                   | Docker Compose Service |
 |----------|---------------------------------------------------------|------------------------|
-| Kafka    | `BROADCAST_URL=kafka://localhost:9092`                  | `docker-compose up kafka` |
-| Redis    | `BROADCAST_URL=redis://localhost:6379`                  | `docker-compose up redis` |
-| Postgres | `BROADCAST_URL=postgres://localhost:5432/broadcaster`   | `docker-compose up postgres` |
-| Pulsar   | `BROADCAST_URL=pulsar://localhost:6650`                 | `docker-compose up pulsar` |
+| Kafka    | `BROADCAST_URL=kafka://localhost:9092`                  | `docker compose up kafka` |
+| Redis    | `BROADCAST_URL=redis://localhost:6379`                  | `docker compose up redis` |
+| Postgres | `BROADCAST_URL=postgres://localhost:5432/broadcaster`   | `docker compose up postgres` |
+| Pulsar   | `BROADCAST_URL=pulsar://localhost:6650`                 | `docker compose up pulsar` |
 
 ## Advanced: Publishing Events Directly to Pulsar
 

diff --git a/documentation/docs/tutorials/track_an_api_bundle_server.mdx b/documentation/docs/tutorials/track_an_api_bundle_server.mdx
@@ -55,7 +55,7 @@ Clone the opal repository and run the example compose file from your local clone
 ```
 git clone https://github.com/permitio/opal.git
 cd opal
-docker-compose -f docker/docker-compose-api-policy-source-example.yml up
+docker compose -f docker/docker-compose-api-policy-source-example.yml up
 ```
 
 The `docker-compose.yml` we just downloaded ([Click here to view its contents](https://github.com/permitio/opal/blob/master/docker/docker-compose-api-policy-source-example.yml)) is running 4 containers: Broadcast, OPAL Server, OPAL Client, and API bundle server.

diff --git a/packages/opal-client/opal_client/config.py b/packages/opal-client/opal_client/config.py
@@ -145,6 +145,12 @@ def load_policy_store():
     # whether or not OPAL should run OPA by itself in the same container
     INLINE_OPA_ENABLED = confi.bool("INLINE_OPA_ENABLED", True)
 
+    INLINE_OPA_EXEC_PATH = confi.str(
+        "INLINE_OPA_EXEC_PATH",
+        None,
+        description="Path to the OPA executable. Defaults to searching for 'opa' binary in PATH if not specified.",
+    )
+
     # if inline OPA is indeed enabled, user can pass cli options
     # (configuration) that affects how OPA will run
     INLINE_OPA_CONFIG = confi.model(
@@ -163,6 +169,12 @@ def load_policy_store():
     # whether or not OPAL should run the Cedar agent by itself in the same container
     INLINE_CEDAR_ENABLED = confi.bool("INLINE_CEDAR_ENABLED", True)
 
+    INLINE_CEDAR_EXEC_PATH = confi.str(
+        "INLINE_CEDAR_EXEC_PATH",
+        None,
+        description="Path to the Cedar Agent executable. Defaults to searching for 'cedar-agent' binary in PATH if not specified.",
+    )
+
     # if inline Cedar is indeed enabled, user can pass cli options
     # (configuration) that affects how the agent will run
     INLINE_CEDAR_CONFIG = confi.model(

diff --git a/packages/opal-client/opal_client/engine/options.py b/packages/opal-client/opal_client/engine/options.py
@@ -1,8 +1,13 @@
+import re
 from enum import Enum
-from typing import Any, List, Optional
+from typing import Any, Iterable, List, Optional
 
 from pydantic import BaseModel, Field, validator
 
+HOST_ADDR_PATTERN = re.compile(
+    r"^(?P<addr>(?:\d{1,3}\.){3}\d{1,3}|)(?::(?P<port>\d+))?$"
+)
+
 
 class LogLevel(str, Enum):
     info = "info"
@@ -77,11 +82,6 @@ def get_cli_options_dict(self):
         """Returns a dict that can be passed to the OPA cli."""
         return self.dict(exclude_none=True, by_alias=True, exclude={"files"})
 
-    def get_opa_startup_files(self) -> str:
-        """Returns a list of startup policies and data."""
-        files = self.files if self.files is not None else []
-        return " ".join(files)
-
 
 class OpenFGAServerOptions(BaseModel):
     """Options to configure the OpenFGA server (apply when choosing to run
@@ -168,7 +168,7 @@ class CedarServerOptions(BaseModel):
     inline)."""
 
     addr: str = Field(
-        ":8181",
+        ":8180",
         description="listening address of the Cedar agent (e.g., [ip]:<port> for TCP)",
     )
     authentication: AuthenticationScheme = Field(
@@ -208,34 +208,23 @@ def validate_authentication_token(cls, v: Optional[str], values: dict[str, Any])
             )
         return v
 
-    def get_cmdline(self) -> str:
-        result = [
-            "cedar-agent",
-        ]
+    def get_args(self) -> Iterable[str]:
         if (
             self.authentication == AuthenticationScheme.token
             and self.authentication_token is not None
         ):
-            result += [
-                "-a",
-                self.authentication_token,
-            ]
-        addr = self.addr.split(":", 1)
-        port = None
-        if len(addr) == 1:
-            listen_address = addr[0]
-        elif len(addr) == 2:
-            listen_address, port = addr
-        if len(listen_address) == 0:
-            listen_address = "0.0.0.0"
-        result += [
-            "--addr",
-            listen_address,
-        ]
-        if port is not None:
-            result += [
-                "--port",
-                port,
-            ]
+            yield "-a"
+            yield self.authentication_token
+
+        match = HOST_ADDR_PATTERN.match(self.addr)
+        if not match:
+            raise ValueError(
+                f"Invalid addr format: {self.addr}. Expected [ip]:<port>, e.g. '0.0.0.0:8180', ':8180'"
+            )
+
+        yield "--addr"
+        yield match.group("addr") or "0.0.0.0"
+        yield "--port"
+        yield match.group("port") or "8180"
+
         # TODO: files
-        return " ".join(result)