help request: Loading Custom Plugins

[WillDera]

Description

I followed the plugin develop documentation to create a new plugin and modify the existing authz-casbin plugin. Below is what my folder structure looks like

This is my apisix conf.yaml file

apisix:
  node_listen: 9080              # APISIX listening port
  enable_ipv6: false
  # Add these lines
  extra_lua_path: "/usr/local/apisix/custom-plugin/?.lua"
  log_level: debug
  enable_control: true
  control:
    ip: "0.0.0.0"
    port: 9092
  

deployment:
  admin:
    allow_admin:               # https://nginx.org/en/docs/http/ngx_http_access_module.html#allow
      - 0.0.0.0/0              # We need to restrict ip access rules for security. 0.0.0.0/0 is for test.

    admin_key:
      - name: "admin"
        key: edd1c9f034335f136f87ad84b625c8f1
        role: admin                 # admin: manage all configuration data

      - name: "viewer"
        key: 4054f7cf07e344346cd3f287985e76a2
        role: viewer

  etcd:
    host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
      - "http://etcd:2379"          # multiple etcd address
    prefix: "/apisix"               # apisix configurations prefix
    timeout: 30                     # 30 seconds

plugin_attr:
  prometheus:
    export_addr:
      ip: "0.0.0.0"
      port: 9091

Here is my apisix dashboard conf.yaml

conf:
  listen:
    host: 0.0.0.0     # `manager api` listening ip or host name
    port: 9000          # `manager api` listening port
  etcd:
    endpoints:          # supports defining multiple etcd host addresses for an etcd cluster
      - etcd:2379

                        # etcd basic auth info
    # username: "root"    # ignore etcd username if not enable etcd auth
    # password: "123456"  # ignore etcd password if not enable etcd auth
  log:
    error_log:
      level: warn       # supports levels, lower to higher: debug, info, warn, error, panic, fatal
      file_path:
        logs/error.log  # supports relative path, absolute path, standard output
                        # such as: logs/error.log, /tmp/logs/error.log, /dev/stdout, /dev/stderr
authentication:
  secret:
    secret              # secret for jwt token generation.
                        # NOTE: Highly recommended to modify this value to protect `manager api`.
                        # if it's default value, when `manager api` start, it will generate a random string to replace it.
  expire_time: 3600     # jwt token expire time, in second
  users:
    - username: admin   # username and password for login `manager api`
      password: admin
    - username: user
      password: user

plugin_attr:
  prometheus:
    export_addr:
      ip: "0.0.0.0"
      port: 9091

plugins:
  - authz-casbin
  - jwt-extractor
  - jwt-auth

Here is my docker-compose.yml file found in example/docker-compose.yml

version: "3"

services:
  apisix:
    image: apache/apisix:${APISIX_IMAGE_TAG:-3.11.0-debian}
    restart: always
    volumes:
      - ./apisix_conf/config.yaml:/usr/local/apisix/conf/config.yaml:ro
      - ./apisix/plugins/custom-plugins:/usr/local/apisix/custom-plugin/apisix/plugins:ro
    depends_on:
      - etcd
    ##network_mode: host
    ports:
      - "9180:9180/tcp"
      - "9080:9080/tcp"
      - "9091:9091/tcp"
      - "9443:9443/tcp"
      - "9092:9092/tcp"
    networks:
      apisix:

  etcd:
    image: bitnami/etcd:3.5.11
    restart: always
    volumes:
      - etcd_data:/bitnami/etcd
    environment:
      ETCD_ENABLE_V2: "true"
      ALLOW_NONE_AUTHENTICATION: "yes"
      ETCD_ADVERTISE_CLIENT_URLS: "http://etcd:2379"
      ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
    ports:
      - "2379:2379/tcp"
    networks:
      apisix:

  web1:
    image: nginx:1.19.0-alpine
    restart: always
    volumes:
      - ./upstream/web1.conf:/etc/nginx/nginx.conf
    ports:
      - "9081:80/tcp"
    environment:
      - NGINX_PORT=80
    networks:
      apisix:

  web2:
    image: nginx:1.19.0-alpine
    restart: always
    volumes:
      - ./upstream/web2.conf:/etc/nginx/nginx.conf
    ports:
      - "9082:80/tcp"
    environment:
      - NGINX_PORT=80
    networks:
      apisix:

  prometheus:
    image: prom/prometheus:v2.25.0
    restart: always
    volumes:
      - ./prometheus_conf/prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"
    networks:
      apisix:

  grafana:
    image: grafana/grafana:7.3.7
    restart: always
    ports:
      - "3000:3000"
    volumes:
      - "./grafana_conf/provisioning:/etc/grafana/provisioning"
      - "./grafana_conf/dashboards:/var/lib/grafana/dashboards"
      - "./grafana_conf/config/grafana.ini:/etc/grafana/grafana.ini"
    networks:
      apisix:

  dashboard:
    image: apache/apisix-dashboard
    restart: always
    ports:
      - "9000:9000"
    volumes:
      - ../all-in-one/apisix-dashboard/conf.yaml:/usr/local/apisix-dashboard/conf/conf.yaml
    networks:
      apisix:
    depends_on:
      - etcd

networks:
  apisix:
    driver: bridge

volumes:
  etcd_data:
    driver: local

The issue here is that I dont see logs in the /usr/local/apisix-dashboard/logs/error.log and /usr/local/apisix-dashboard/logs/access.log nor in /usr/local/apisix/logs/error.log.

I'm not sure what I'm doing wrongly. I can see the plugin files are properly copied into the apisix server container in the folder I described:

❯ docker exec docker-apisix-apisix-1 ls /usr/local/apisix/custom-plugin/apisix/plugins
authz-casbin.lua
jwt-extractor.lua

but when I make the following curl request

curl --request PUT \
  --url http://127.0.0.1:9180/apisix/admin/routes \
  --header 'Content-Type: application/json' \
  --header 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' \
  --data '{
  "id":"jwt-extractor",
  "uri":"/test",
  "plugins": {
		"jwt-extractor": {
			"role_claim": "role"
		}
	}
}'

I get the following response back

{
	"error_msg": "unknown plugin [jwt-extractor]"
}

Environment

• APISIX version (run apisix version): 3.11.0-debian
• Operating system (run uname -a): debian (docker image)
• OpenResty / Nginx version (run openresty -V or nginx -V): 1.19,0
• etcd version, if relevant (run curl http://127.0.0.1:9090/v1/server_info): 3.5.0 (the curl request returns 404)
• APISIX Dashboard version, if relevant: 3.0.1
• Plugin runner version, for issues related to plugin runners: Not Sure where to get this
• LuaRocks version, for installation issues (run luarocks --version): Not Sure where to get this

[mikyll]

You're facing this issue because you only set the plugin list for the dashboard.

You should append this to apisix_conf/config.yaml:

plugins:
  - authz-casbin
  - jwt-extractor
  - jwt-auth

However, be aware that this overwrites the default plugin list. Typically you would take the default list from apache/apisix/conf/config.yaml.example and add your custom plugins there, e.g.:

plugins:                           # plugin list (sorted by priority)
  - real-ip                        # priority: 23000
  - ai                             # priority: 22900
  - client-control                 # priority: 22000
  - proxy-control                  # priority: 21990
  - request-id                     # priority: 12015
  - zipkin                         # priority: 12011
  #- skywalking                    # priority: 12010
  #- opentelemetry                 # priority: 12009
  - ext-plugin-pre-req             # priority: 12000
  - fault-injection                # priority: 11000
  - mocking                        # priority: 10900
  - serverless-pre-function        # priority: 10000
  #- batch-requests                # priority: 4010
  - cors                           # priority: 4000
  - ip-restriction                 # priority: 3000
  - ua-restriction                 # priority: 2999
  - referer-restriction            # priority: 2990
  - csrf                           # priority: 2980
  - uri-blocker                    # priority: 2900
  - request-validation             # priority: 2800
  - chaitin-waf                    # priority: 2700
  - multi-auth                     # priority: 2600
  - openid-connect                 # priority: 2599
  - cas-auth                       # priority: 2597
  - authz-casbin                   # priority: 2560
  - authz-casdoor                  # priority: 2559
  - wolf-rbac                      # priority: 2555
  - ldap-auth                      # priority: 2540
  - hmac-auth                      # priority: 2530
  - basic-auth                     # priority: 2520
  - jwt-auth                       # priority: 2510
  - jwe-decrypt                    # priority: 2509
  - key-auth                       # priority: 2500
  - consumer-restriction           # priority: 2400
  - attach-consumer-label          # priority: 2399
  - forward-auth                   # priority: 2002
  - opa                            # priority: 2001
  - authz-keycloak                 # priority: 2000
  #- error-log-logger              # priority: 1091
  - proxy-cache                    # priority: 1085
  - body-transformer               # priority: 1080
  - ai-prompt-template             # priority: 1071
  - ai-prompt-decorator            # priority: 1070
  - ai-rag                         # priority: 1060
  - ai-content-moderation          # priority: 1040 TODO: compare priority with other ai plugins
  - proxy-mirror                   # priority: 1010
  - proxy-rewrite                  # priority: 1008
  - workflow                       # priority: 1006
  - api-breaker                    # priority: 1005
  - limit-conn                     # priority: 1003
  - limit-count                    # priority: 1002
  - limit-req                      # priority: 1001
  #- node-status                   # priority: 1000
  - ai-proxy                       # priority: 999
  #- brotli                        # priority: 996
  - gzip                           # priority: 995
  - server-info                    # priority: 990
  - traffic-split                  # priority: 966
  - redirect                       # priority: 900
  - response-rewrite               # priority: 899
  - degraphql                      # priority: 509
  - kafka-proxy                    # priority: 508
  #- dubbo-proxy                   # priority: 507
  - grpc-transcode                 # priority: 506
  - grpc-web                       # priority: 505
  - http-dubbo                     # priority: 504
  - public-api                     # priority: 501
  - prometheus                     # priority: 500
  - datadog                        # priority: 495
  - loki-logger                    # priority: 414
  - elasticsearch-logger           # priority: 413
  - echo                           # priority: 412
  - loggly                         # priority: 411
  - http-logger                    # priority: 410
  - splunk-hec-logging             # priority: 409
  - skywalking-logger              # priority: 408
  - google-cloud-logging           # priority: 407
  - sls-logger                     # priority: 406
  - tcp-logger                     # priority: 405
  - kafka-logger                   # priority: 403
  - rocketmq-logger                # priority: 402
  - syslog                         # priority: 401
  - udp-logger                     # priority: 400
  - file-logger                    # priority: 399
  - clickhouse-logger              # priority: 398
  - tencent-cloud-cls              # priority: 397
  - inspect                        # priority: 200
  #- log-rotate                    # priority: 100
  # <- recommend to use priority (0, 100) for your custom plugins
  - jwt-extractor
  
  - example-plugin                 # priority: 0
  #- gm                            # priority: -43
  #- ocsp-stapling                 # priority: -44
  - aws-lambda                     # priority: -1899
  - azure-functions                # priority: -1900
  - openwhisk                      # priority: -1901
  - openfunction                   # priority: -1902
  - serverless-post-function       # priority: -2000
  - ext-plugin-post-req            # priority: -3000
  - ext-plugin-post-resp           # priority: -4000

Let me know if that solved the issue 🙂