diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 09fb90c10..e2083cdd1 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -31,6 +31,7 @@ body: label: Version description: What version are you running? options: + - v0.30.0 - v0.26.0 - v0.25.0 - v0.24.3 diff --git a/.github/workflows/deletedroplets.yml b/.github/workflows/deletedroplets.yml index 9fc1834f9..f3412e2ef 100644 --- a/.github/workflows/deletedroplets.yml +++ b/.github/workflows/deletedroplets.yml @@ -12,7 +12,7 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: get logs - uses: dawidd6/action-download-artifact@v6 + uses: dawidd6/action-download-artifact@v7 with: run_id: ${{ github.event.workflow_run.id}} if_no_artifact_found: warn @@ -75,7 +75,7 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - name: get logs - uses: dawidd6/action-download-artifact@v6 + uses: dawidd6/action-download-artifact@v7 with: run_id: ${{ github.event.workflow_run.id}} if_no_artifact_found: warn diff --git a/.swaggo b/.swaggo new file mode 100644 index 000000000..3f4cefc27 --- /dev/null +++ b/.swaggo @@ -0,0 +1,2 @@ +// Replace all time.Duration with int64 +replace time.Duration int64 diff --git a/Dockerfile b/Dockerfile index f012ac3fe..3dd1a0e9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN GOOS=linux CGO_ENABLED=1 go build -ldflags="-s -w " -tags ${tags} . # RUN go build -tags=ee . -o netmaker main.go -FROM alpine:3.20.3 +FROM alpine:3.21.0 # add a c lib # set the working directory diff --git a/Dockerfile-quick b/Dockerfile-quick index 10f4f9808..4180634a4 100644 --- a/Dockerfile-quick +++ b/Dockerfile-quick @@ -1,5 +1,5 @@ #first stage - builder -FROM alpine:3.20.3 +FROM alpine:3.21.0 ARG version WORKDIR /app COPY ./netmaker /root/netmaker diff --git a/README.md b/README.md index 137d66284..6e0a9a04e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@

- + diff --git a/auth/host_session.go b/auth/host_session.go index 7a3929240..4de8d6630 100644 --- a/auth/host_session.go +++ b/auth/host_session.go @@ -257,7 +257,7 @@ func CheckNetRegAndHostUpdate(networks []string, h *models.Host, relayNodeId uui if relayNodeId != uuid.Nil && !newNode.IsRelayed { // check if relay node exists and acting as relay relaynode, err := logic.GetNodeByID(relayNodeId.String()) - if err == nil && relaynode.IsRelay { + if err == nil && relaynode.IsRelay && relaynode.Network == newNode.Network { slog.Info(fmt.Sprintf("adding relayed node %s to relay %s on network %s", newNode.ID.String(), relayNodeId.String(), network)) newNode.IsRelayed = true newNode.RelayedBy = relayNodeId.String() @@ -271,7 +271,7 @@ func CheckNetRegAndHostUpdate(networks []string, h *models.Host, relayNodeId uui slog.Error("failed to update node", "nodeid", relayNodeId.String()) } } else { - slog.Error("failed to relay node. maybe specified relay node is actually not a relay?", "err", err) + slog.Error("failed to relay node. maybe specified relay node is actually not a relay? Or the relayed node is not in the same network with relay?", "err", err) } } logger.Log(1, "added new node", newNode.ID.String(), "to host", h.Name) diff --git a/compose/docker-compose-emqx.yml b/compose/docker-compose-emqx.yml index 61dabf08e..94de8fad9 100644 --- a/compose/docker-compose-emqx.yml +++ b/compose/docker-compose-emqx.yml @@ -3,7 +3,7 @@ version: "3.4" services: mq: container_name: mq - image: emqx/emqx:5.0.9 + image: emqx/emqx:5.8.2 env_file: ./netmaker.env restart: unless-stopped environment: @@ -20,6 +20,7 @@ services: - emqx_data:/opt/emqx/data - emqx_etc:/opt/emqx/etc - emqx_logs:/opt/emqx/log + - ./emqx.conf:/opt/emqx/data/configs/cluster.hocon volumes: emqx_data: { } # storage for emqx data emqx_etc: { } # storage for emqx etc diff --git a/compose/docker-compose.netclient.yml b/compose/docker-compose.netclient.yml index da6fc14f4..bb7c40347 100644 --- a/compose/docker-compose.netclient.yml +++ b/compose/docker-compose.netclient.yml @@ -3,7 +3,7 @@ version: "3.4" services: netclient: container_name: netclient - image: 'gravitl/netclient:v0.26.0' + image: 'gravitl/netclient:v0.30.0' hostname: netmaker-1 network_mode: host restart: on-failure diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index e32481393..879b38ded 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -12,7 +12,7 @@ services: - sqldata:/root/data environment: # config-dependant vars - - STUN_LIST=stun1.netmaker.io:3478,stun2.netmaker.io:3478,stun1.l.google.com:19302,stun2.l.google.com:19302 + - STUN_SERVERS=stun1.netmaker.io:3478,stun2.netmaker.io:3478,stun1.l.google.com:19302,stun2.l.google.com:19302 # The domain/host IP indicating the mq broker address - BROKER_ENDPOINT=wss://broker.${NM_DOMAIN} # For EMQX broker use `BROKER_ENDPOINT=wss://broker.${NM_DOMAIN}/mqtt` # For EMQX broker (uncomment the two lines below) @@ -52,8 +52,8 @@ services: - caddy_data:/data - caddy_conf:/config ports: - - "80:80" - - "443:443" + - "$SERVER_HOST:80:80" + - "$SERVER_HOST:443:443" coredns: #network_mode: host diff --git a/config/config.go b/config/config.go index f9acaf980..36ce1cbf1 100644 --- a/config/config.go +++ b/config/config.go @@ -89,7 +89,7 @@ type ServerConfig struct { EgressesLimit int `yaml:"egresses_limit"` DeployedByOperator bool `yaml:"deployed_by_operator"` Environment string `yaml:"environment"` - JwtValidityDuration time.Duration `yaml:"jwt_validity_duration"` + JwtValidityDuration time.Duration `yaml:"jwt_validity_duration" swaggertype:"primitive,integer" format:"int64"` RacAutoDisable bool `yaml:"rac_auto_disable"` CacheEnabled string `yaml:"caching_enabled"` EndpointDetection bool `json:"endpoint_detection"` @@ -101,7 +101,10 @@ type ServerConfig struct { SmtpPort int `json:"smtp_port"` MetricInterval string `yaml:"metric_interval"` ManageDNS bool `yaml:"manage_dns"` + Stun bool `yaml:"stun"` + StunServers string `yaml:"stun_servers"` DefaultDomain string `yaml:"default_domain"` + PublicIp string `yaml:"public_ip"` } // SQLConfig - Generic SQL Config diff --git a/controllers/acls.go b/controllers/acls.go index 727811fb5..2871aef55 100644 --- a/controllers/acls.go +++ b/controllers/acls.go @@ -52,6 +52,81 @@ func aclPolicyTypes(w http.ResponseWriter, r *http.Request) { // models.NetmakerIPAclID, // models.NetmakerSubNetRangeAClID, }, + ProtocolTypes: []models.ProtocolType{ + { + Name: models.Any, + AllowedProtocols: []models.Protocol{ + models.ALL, + }, + PortRange: "All ports", + AllowPortSetting: false, + }, + { + Name: models.Http, + AllowedProtocols: []models.Protocol{ + models.TCP, + }, + PortRange: "80", + }, + { + Name: models.Https, + AllowedProtocols: []models.Protocol{ + models.TCP, + }, + PortRange: "443", + }, + // { + // Name: "MySQL", + // AllowedProtocols: []models.Protocol{ + // models.TCP, + // }, + // PortRange: "3306", + // }, + // { + // Name: "DNS TCP", + // AllowedProtocols: []models.Protocol{ + // models.TCP, + // }, + // PortRange: "53", + // }, + // { + // Name: "DNS UDP", + // AllowedProtocols: []models.Protocol{ + // models.UDP, + // }, + // PortRange: "53", + // }, + { + Name: models.AllTCP, + AllowedProtocols: []models.Protocol{ + models.TCP, + }, + PortRange: "All ports", + }, + { + Name: models.AllUDP, + AllowedProtocols: []models.Protocol{ + models.UDP, + }, + PortRange: "All ports", + }, + { + Name: models.ICMPService, + AllowedProtocols: []models.Protocol{ + models.ICMP, + }, + PortRange: "", + }, + { + Name: models.Custom, + AllowedProtocols: []models.Protocol{ + models.UDP, + models.TCP, + }, + PortRange: "All ports", + AllowPortSetting: true, + }, + }, } logic.ReturnSuccessResponseWithJson(w, r, resp, "fetched acls types") } @@ -69,7 +144,7 @@ func aclDebug(w http.ResponseWriter, r *http.Request) { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - allowed := logic.IsNodeAllowedToCommunicate(node, peer) + allowed, _ := logic.IsNodeAllowedToCommunicate(node, peer, true) logic.ReturnSuccessResponseWithJson(w, r, allowed, "fetched all acls in the network ") } @@ -91,7 +166,7 @@ func getAcls(w http.ResponseWriter, r *http.Request) { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - acls, err := logic.ListAcls(models.NetworkID(netID)) + acls, err := logic.ListAclsByNetwork(models.NetworkID(netID)) if err != nil { logger.Log(0, r.Header.Get("user"), "failed to get all network acl entries: ", err.Error()) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) @@ -132,10 +207,9 @@ func createAcl(w http.ResponseWriter, r *http.Request) { acl.CreatedBy = user.UserName acl.CreatedAt = time.Now().UTC() acl.Default = false - if acl.RuleType == models.DevicePolicy { - acl.AllowedDirection = models.TrafficDirectionBi - } else { - acl.AllowedDirection = models.TrafficDirectionUni + if acl.ServiceType == models.Any { + acl.Port = []string{} + acl.Proto = models.ALL } // validate create acl policy if !logic.IsAclPolicyValid(acl) { @@ -152,7 +226,7 @@ func createAcl(w http.ResponseWriter, r *http.Request) { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return } - go mq.PublishPeerUpdate(false) + go mq.PublishPeerUpdate(true) logic.ReturnSuccessResponseWithJson(w, r, acl, "created acl successfully") } @@ -194,7 +268,7 @@ func updateAcl(w http.ResponseWriter, r *http.Request) { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - go mq.PublishPeerUpdate(false) + go mq.PublishPeerUpdate(true) logic.ReturnSuccessResponse(w, r, "updated acl "+acl.Name) } @@ -225,6 +299,6 @@ func deleteAcl(w http.ResponseWriter, r *http.Request) { logic.FormatError(errors.New("cannot delete default policy"), "internal")) return } - go mq.PublishPeerUpdate(false) + go mq.PublishPeerUpdate(true) logic.ReturnSuccessResponse(w, r, "deleted acl "+acl.Name) } diff --git a/controllers/dns.go b/controllers/dns.go index cc1d70abf..67085365f 100644 --- a/controllers/dns.go +++ b/controllers/dns.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strings" "github.com/gorilla/mux" "github.com/gravitl/netmaker/database" @@ -162,7 +163,10 @@ func createDNS(w http.ResponseWriter, r *http.Request) { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - + // check if default domain is appended if not append + if !strings.HasSuffix(entry.Name, servercfg.GetDefaultDomain()) { + entry.Name += "." + servercfg.GetDefaultDomain() + } entry, err = logic.CreateDNS(entry) if err != nil { logger.Log(0, r.Header.Get("user"), diff --git a/controllers/dns_test.go b/controllers/dns_test.go index 4682f5258..ccce338c6 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -391,7 +391,7 @@ func TestValidateDNSCreate(t *testing.T) { entry := models.DNSEntry{Address: "10.0.0.2", Network: "skynet"} err := logic.ValidateDNSCreate(entry) assert.NotNil(t, err) - assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'required' tag") + assert.Contains(t, err.Error(), "invalid input") }) t.Run("NameTooLong", func(t *testing.T) { name := "" @@ -414,13 +414,13 @@ func TestValidateDNSCreate(t *testing.T) { entry := models.DNSEntry{Address: "10.10.10.5", Name: "white space", Network: "skynet"} err := logic.ValidateDNSCreate(entry) assert.NotNil(t, err) - assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'whitespace' tag") + assert.Contains(t, err.Error(), "invalid input") }) t.Run("AllSpaces", func(t *testing.T) { entry := models.DNSEntry{Address: "10.10.10.5", Name: " ", Network: "skynet"} err := logic.ValidateDNSCreate(entry) assert.NotNil(t, err) - assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'whitespace' tag") + assert.Contains(t, err.Error(), "invalid input") }) } diff --git a/controllers/ext_client.go b/controllers/ext_client.go index 874c62576..1c751cc47 100644 --- a/controllers/ext_client.go +++ b/controllers/ext_client.go @@ -490,6 +490,13 @@ func createExtClient(w http.ResponseWriter, r *http.Request) { if err == nil { // check if parent network default ACL is enabled (yes) or not (no) extclient.Enabled = parentNetwork.DefaultACL == "yes" } + extclient.Os = customExtClient.Os + extclient.DeviceName = customExtClient.DeviceName + if customExtClient.IsAlreadyConnectedToInetGw { + slog.Warn("RAC/Client is already connected to internet gateway. this may mask their real IP address", "client IP", customExtClient.PublicEndpoint) + } + extclient.PublicEndpoint = customExtClient.PublicEndpoint + extclient.Country = customExtClient.Country if err = logic.CreateExtClient(&extclient); err != nil { slog.Error( diff --git a/controllers/node.go b/controllers/node.go index d7f2e2569..904f6375b 100644 --- a/controllers/node.go +++ b/controllers/node.go @@ -326,8 +326,9 @@ func getNetworkNodes(w http.ResponseWriter, r *http.Request) { if len(filteredNodes) > 0 { nodes = filteredNodes } - nodes = logic.AddStaticNodestoList(nodes) + nodes = logic.AddStaticNodestoList(nodes) + nodes = logic.AddStatusToNodes(nodes) // returns all the nodes in JSON/API format apiNodes := logic.GetAllNodesAPI(nodes[:]) logger.Log(2, r.Header.Get("user"), "fetched nodes on network", networkName) @@ -367,6 +368,7 @@ func getAllNodes(w http.ResponseWriter, r *http.Request) { } nodes = logic.AddStaticNodestoList(nodes) + nodes = logic.AddStatusToNodes(nodes) // return all the nodes in JSON/API format apiNodes := logic.GetAllNodesAPI(nodes[:]) logger.Log(3, r.Header.Get("user"), "fetched all nodes they have access to") @@ -679,6 +681,11 @@ func updateNode(w http.ResponseWriter, r *http.Request) { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } + err = logic.ValidateNodeIp(¤tNode, &newData) + if err != nil { + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) + return + } if !servercfg.IsPro { newData.AdditionalRagIps = []string{} } diff --git a/controllers/server.go b/controllers/server.go index 77b06fcc7..171bbdb32 100644 --- a/controllers/server.go +++ b/controllers/server.go @@ -48,6 +48,8 @@ func serverHandlers(r *mux.Router) { Methods(http.MethodGet) r.HandleFunc("/api/server/cpu_profile", logic.SecurityCheck(false, http.HandlerFunc(cpuProfile))). Methods(http.MethodPost) + r.HandleFunc("/api/server/mem_profile", logic.SecurityCheck(false, http.HandlerFunc(memProfile))). + Methods(http.MethodPost) } func cpuProfile(w http.ResponseWriter, r *http.Request) { @@ -62,6 +64,10 @@ func cpuProfile(w http.ResponseWriter, r *http.Request) { } } } +func memProfile(w http.ResponseWriter, r *http.Request) { + os.Remove("/root/data/mem.prof") + logic.StartMemProfiling() +} func getUsage(w http.ResponseWriter, _ *http.Request) { type usage struct { diff --git a/controllers/user.go b/controllers/user.go index 8f3375cc8..02ba98aee 100644 --- a/controllers/user.go +++ b/controllers/user.go @@ -722,7 +722,7 @@ func socketHandler(w http.ResponseWriter, r *http.Request) { // @Summary lists all user roles. // @Router /api/v1/user/roles [get] // @Tags Users -// @Param role_id param string true "roleid required to get the role details" +// @Param role_id query string true "roleid required to get the role details" // @Success 200 {object} []models.UserRolePermissionTemplate // @Failure 500 {object} models.ErrorResponse func listRoles(w http.ResponseWriter, r *http.Request) { diff --git a/docker/emqx.conf b/docker/emqx.conf new file mode 100644 index 000000000..e0af69c2b --- /dev/null +++ b/docker/emqx.conf @@ -0,0 +1,21 @@ +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm { + name = "sha256", + salt_position = "suffix" + } + user_id_type = "username" + } +] +authorization { + deny_action = ignore + no_match = allow + sources = [ + { + type = built_in_database + enable = true + } + ] +} diff --git a/go.mod b/go.mod index b337d784a..b06a2b954 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module github.com/gravitl/netmaker go 1.23 require ( + github.com/blang/semver v3.5.1+incompatible github.com/eclipse/paho.mqtt.golang v1.4.3 - github.com/go-playground/validator/v10 v10.22.1 - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/go-playground/validator/v10 v10.23.0 + github.com/golang-jwt/jwt/v4 v4.5.1 github.com/google/uuid v1.6.0 github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 @@ -14,13 +15,14 @@ require ( github.com/rqlite/gorqlite v0.0.0-20240122221808-a8a425b1a6aa github.com/seancfoley/ipaddress-go v1.7.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/txn2/txeh v1.5.5 - golang.org/x/crypto v0.28.0 + go.uber.org/automaxprocs v1.6.0 + golang.org/x/crypto v0.30.0 golang.org/x/net v0.27.0 // indirect - golang.org/x/oauth2 v0.23.0 - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect + golang.org/x/oauth2 v0.24.0 + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb gopkg.in/yaml.v3 v3.0.1 ) @@ -51,6 +53,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/seancfoley/bintree v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -66,5 +69,5 @@ require ( github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sync v0.8.0 // indirect + golang.org/x/sync v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index b59242e92..a25549d4e 100644 --- a/go.sum +++ b/go.sum @@ -2,11 +2,14 @@ cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2Qx cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/c-robinson/iplib v1.0.8 h1:exDRViDyL9UBLcfmlxxkY5odWX5092nPsQIykHXhIn4= github.com/c-robinson/iplib v1.0.8/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo= github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo= github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -24,10 +27,10 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= +github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -46,6 +49,10 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -64,6 +71,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posthog/posthog-go v1.2.24 h1:A+iG4saBJemo++VDlcWovbYf8KFFNUfrCoJtsc40RPA= github.com/posthog/posthog-go v1.2.24/go.mod h1:uYC2l1Yktc8E+9FAHJ9QZG4vQf/NHJPD800Hsm7DzoM= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -82,16 +91,18 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/txn2/txeh v1.5.5 h1:UN4e/lCK5HGw/gGAi2GCVrNKg0GTCUWs7gs5riaZlz4= github.com/txn2/txeh v1.5.5/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -103,13 +114,13 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -118,8 +129,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -131,8 +142,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -142,8 +153,9 @@ golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb h1:9aqVcYED golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb/go.mod h1:mQqgjkW8GQQcJQsbBvK890TKqUK1DfKWkuBGbOkuMHQ= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk= gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/k8s/client/netclient-daemonset.yaml b/k8s/client/netclient-daemonset.yaml index cc7b744a0..3eff2e162 100644 --- a/k8s/client/netclient-daemonset.yaml +++ b/k8s/client/netclient-daemonset.yaml @@ -16,7 +16,7 @@ spec: hostNetwork: true containers: - name: netclient - image: gravitl/netclient:v0.26.0 + image: gravitl/netclient:v0.30.0 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/client/netclient.yaml b/k8s/client/netclient.yaml index 9aa3f396d..3f5f4a45a 100644 --- a/k8s/client/netclient.yaml +++ b/k8s/client/netclient.yaml @@ -28,7 +28,7 @@ spec: # - "" containers: - name: netclient - image: gravitl/netclient:v0.26.0 + image: gravitl/netclient:v0.30.0 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/server/netmaker-ui.yaml b/k8s/server/netmaker-ui.yaml index 3212ec10a..43b7f98d2 100644 --- a/k8s/server/netmaker-ui.yaml +++ b/k8s/server/netmaker-ui.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: netmaker-ui - image: gravitl/netmaker-ui:v0.26.0 + image: gravitl/netmaker-ui:v0.30.0 ports: - containerPort: 443 env: diff --git a/logic/acls.go b/logic/acls.go index 9a4dcb4aa..cf0c4f87d 100644 --- a/logic/acls.go +++ b/logic/acls.go @@ -18,12 +18,25 @@ var ( aclCacheMap = make(map[string]models.Acl) ) +func MigrateAclPolicies() { + acls := ListAcls() + for _, acl := range acls { + if acl.Proto.String() == "" { + acl.Proto = models.ALL + acl.ServiceType = models.Any + acl.Port = []string{} + UpsertAcl(acl) + } + } + +} + // CreateDefaultAclNetworkPolicies - create default acl network policies func CreateDefaultAclNetworkPolicies(netID models.NetworkID) { if netID.String() == "" { return } - _, _ = ListAcls(netID) + _, _ = ListAclsByNetwork(netID) if !IsAclExists(fmt.Sprintf("%s.%s", netID, "all-nodes")) { defaultDeviceAcl := models.Acl{ ID: fmt.Sprintf("%s.%s", netID, "all-nodes"), @@ -31,6 +44,8 @@ func CreateDefaultAclNetworkPolicies(netID models.NetworkID) { MetaData: "This Policy allows all nodes in the network to communicate with each other", Default: true, NetworkID: netID, + Proto: models.ALL, + Port: []string{}, RuleType: models.DevicePolicy, Src: []models.AclPolicyTag{ { @@ -56,6 +71,8 @@ func CreateDefaultAclNetworkPolicies(netID models.NetworkID) { Name: "All Users", MetaData: "This policy gives access to everything in the network for an user", NetworkID: netID, + Proto: models.ALL, + Port: []string{}, RuleType: models.UserPolicy, Src: []models.AclPolicyTag{ { @@ -81,6 +98,8 @@ func CreateDefaultAclNetworkPolicies(netID models.NetworkID) { Default: true, Name: "All Remote Access Gateways", NetworkID: netID, + Proto: models.ALL, + Port: []string{}, RuleType: models.DevicePolicy, Src: []models.AclPolicyTag{ { @@ -106,7 +125,7 @@ func CreateDefaultAclNetworkPolicies(netID models.NetworkID) { // DeleteDefaultNetworkPolicies - deletes all default network acl policies func DeleteDefaultNetworkPolicies(netId models.NetworkID) { - acls, _ := ListAcls(netId) + acls, _ := ListAclsByNetwork(netId) for _, acl := range acls { if acl.NetworkID == netId && acl.Default { DeleteAcl(acl) @@ -202,7 +221,10 @@ func IsAclExists(aclID string) bool { // IsAclPolicyValid - validates if acl policy is valid func IsAclPolicyValid(acl models.Acl) bool { //check if src and dst are valid - + if acl.AllowedDirection != models.TrafficDirectionBi && + acl.AllowedDirection != models.TrafficDirectionUni { + return false + } switch acl.RuleType { case models.UserPolicy: // src list should only contain users @@ -298,6 +320,14 @@ func UpdateAcl(newAcl, acl models.Acl) error { acl.Name = newAcl.Name acl.Src = newAcl.Src acl.Dst = newAcl.Dst + acl.AllowedDirection = newAcl.AllowedDirection + acl.Port = newAcl.Port + acl.Proto = newAcl.Proto + acl.ServiceType = newAcl.ServiceType + } + if newAcl.ServiceType == models.Any { + acl.Port = []string{} + acl.Proto = models.ALL } acl.Enabled = newAcl.Enabled d, err := json.Marshal(acl) @@ -347,14 +377,20 @@ func GetDefaultPolicy(netID models.NetworkID, ruleType models.AclPolicyType) (mo return acl, nil } // check if there are any custom all policies - policies, _ := ListAcls(netID) + srcMap := make(map[string]struct{}) + dstMap := make(map[string]struct{}) + defer func() { + srcMap = nil + dstMap = nil + }() + policies, _ := ListAclsByNetwork(netID) for _, policy := range policies { if !policy.Enabled { continue } if policy.RuleType == ruleType { - dstMap := convAclTagToValueMap(policy.Dst) - srcMap := convAclTagToValueMap(policy.Src) + dstMap = convAclTagToValueMap(policy.Dst) + srcMap = convAclTagToValueMap(policy.Src) if _, ok := srcMap["*"]; ok { if _, ok := dstMap["*"]; ok { return policy, nil @@ -367,7 +403,7 @@ func GetDefaultPolicy(netID models.NetworkID, ruleType models.AclPolicyType) (mo return acl, nil } -func listAcls() (acls []models.Acl) { +func ListAcls() (acls []models.Acl) { if servercfg.CacheEnabled() && len(aclCacheMap) > 0 { return listAclFromCache() } @@ -376,7 +412,6 @@ func listAcls() (acls []models.Acl) { if err != nil && !database.IsEmptyRecord(err) { return []models.Acl{} } - for _, dataI := range data { acl := models.Acl{} err := json.Unmarshal([]byte(dataI), &acl) @@ -393,7 +428,7 @@ func listAcls() (acls []models.Acl) { // ListUserPolicies - lists all acl policies enforced on an user func ListUserPolicies(u models.User) []models.Acl { - allAcls := listAcls() + allAcls := ListAcls() userAcls := []models.Acl{} for _, acl := range allAcls { @@ -418,7 +453,7 @@ func ListUserPolicies(u models.User) []models.Acl { // listPoliciesOfUser - lists all user acl policies applied to user in an network func listPoliciesOfUser(user models.User, netID models.NetworkID) []models.Acl { - allAcls := listAcls() + allAcls := ListAcls() userAcls := []models.Acl{} for _, acl := range allAcls { if acl.NetworkID == netID && acl.RuleType == models.UserPolicy { @@ -447,7 +482,7 @@ func listPoliciesOfUser(user models.User, netID models.NetworkID) []models.Acl { // listDevicePolicies - lists all device policies in a network func listDevicePolicies(netID models.NetworkID) []models.Acl { - allAcls := listAcls() + allAcls := ListAcls() deviceAcls := []models.Acl{} for _, acl := range allAcls { if acl.NetworkID == netID && acl.RuleType == models.DevicePolicy { @@ -457,10 +492,22 @@ func listDevicePolicies(netID models.NetworkID) []models.Acl { return deviceAcls } +// listUserPolicies - lists all user policies in a network +func listUserPolicies(netID models.NetworkID) []models.Acl { + allAcls := ListAcls() + deviceAcls := []models.Acl{} + for _, acl := range allAcls { + if acl.NetworkID == netID && acl.RuleType == models.UserPolicy { + deviceAcls = append(deviceAcls, acl) + } + } + return deviceAcls +} + // ListAcls - lists all acl policies -func ListAcls(netID models.NetworkID) ([]models.Acl, error) { +func ListAclsByNetwork(netID models.NetworkID) ([]models.Acl, error) { - allAcls := listAcls() + allAcls := ListAcls() netAcls := []models.Acl{} for _, acl := range allAcls { if acl.NetworkID == netID { @@ -479,19 +526,19 @@ func convAclTagToValueMap(acltags []models.AclPolicyTag) map[string]struct{} { } // IsUserAllowedToCommunicate - check if user is allowed to communicate with peer -func IsUserAllowedToCommunicate(userName string, peer models.Node) bool { +func IsUserAllowedToCommunicate(userName string, peer models.Node) (bool, []models.Acl) { if peer.IsStatic { peer = peer.StaticNode.ConvertToStaticNode() } acl, _ := GetDefaultPolicy(models.NetworkID(peer.Network), models.UserPolicy) if acl.Enabled { - return true + return true, []models.Acl{acl} } user, err := GetUser(userName) if err != nil { - return false + return false, []models.Acl{} } - + allowedPolicies := []models.Acl{} policies := listPoliciesOfUser(*user, models.NetworkID(peer.Network)) for _, policy := range policies { if !policy.Enabled { @@ -499,46 +546,54 @@ func IsUserAllowedToCommunicate(userName string, peer models.Node) bool { } dstMap := convAclTagToValueMap(policy.Dst) if _, ok := dstMap["*"]; ok { - return true + allowedPolicies = append(allowedPolicies, policy) + continue } for tagID := range peer.Tags { if _, ok := dstMap[tagID.String()]; ok { - return true + allowedPolicies = append(allowedPolicies, policy) + break } } } - return false + if len(allowedPolicies) > 0 { + return true, allowedPolicies + } + return false, []models.Acl{} } -// IsNodeAllowedToCommunicate - check node is allowed to communicate with the peer -func IsNodeAllowedToCommunicate(node, peer models.Node) bool { +// IsPeerAllowed - checks if peer needs to be added to the interface +func IsPeerAllowed(node, peer models.Node, checkDefaultPolicy bool) bool { if node.IsStatic { node = node.StaticNode.ConvertToStaticNode() } if peer.IsStatic { peer = peer.StaticNode.ConvertToStaticNode() } - // check default policy if all allowed return true - defaultPolicy, err := GetDefaultPolicy(models.NetworkID(node.Network), models.DevicePolicy) - if err == nil { - if defaultPolicy.Enabled { - return true + if checkDefaultPolicy { + // check default policy if all allowed return true + defaultPolicy, err := GetDefaultPolicy(models.NetworkID(node.Network), models.DevicePolicy) + if err == nil { + if defaultPolicy.Enabled { + return true + } } } - // list device policies policies := listDevicePolicies(models.NetworkID(peer.Network)) + srcMap := make(map[string]struct{}) + dstMap := make(map[string]struct{}) + defer func() { + srcMap = nil + dstMap = nil + }() for _, policy := range policies { if !policy.Enabled { continue } - srcMap := convAclTagToValueMap(policy.Src) - dstMap := convAclTagToValueMap(policy.Dst) - // fmt.Printf("\n======> SRCMAP: %+v\n", srcMap) - // fmt.Printf("\n======> DSTMAP: %+v\n", dstMap) - // fmt.Printf("\n======> node Tags: %+v\n", node.Tags) - // fmt.Printf("\n======> peer Tags: %+v\n", peer.Tags) + srcMap = convAclTagToValueMap(policy.Src) + dstMap = convAclTagToValueMap(policy.Dst) for tagID := range node.Tags { if _, ok := dstMap[tagID.String()]; ok { if _, ok := srcMap["*"]; ok { @@ -588,6 +643,122 @@ func IsNodeAllowedToCommunicate(node, peer models.Node) bool { return false } +// IsNodeAllowedToCommunicate - check node is allowed to communicate with the peer +func IsNodeAllowedToCommunicate(node, peer models.Node, checkDefaultPolicy bool) (bool, []models.Acl) { + if node.IsStatic { + node = node.StaticNode.ConvertToStaticNode() + } + if peer.IsStatic { + peer = peer.StaticNode.ConvertToStaticNode() + } + if checkDefaultPolicy { + // check default policy if all allowed return true + defaultPolicy, err := GetDefaultPolicy(models.NetworkID(node.Network), models.DevicePolicy) + if err == nil { + if defaultPolicy.Enabled { + return true, []models.Acl{defaultPolicy} + } + } + } + allowedPolicies := []models.Acl{} + // list device policies + policies := listDevicePolicies(models.NetworkID(peer.Network)) + srcMap := make(map[string]struct{}) + dstMap := make(map[string]struct{}) + defer func() { + srcMap = nil + dstMap = nil + }() + for _, policy := range policies { + if !policy.Enabled { + continue + } + srcMap = convAclTagToValueMap(policy.Src) + dstMap = convAclTagToValueMap(policy.Dst) + for tagID := range node.Tags { + allowed := false + if _, ok := dstMap[tagID.String()]; policy.AllowedDirection == models.TrafficDirectionBi && ok { + if _, ok := srcMap["*"]; ok { + allowed = true + allowedPolicies = append(allowedPolicies, policy) + break + } + for tagID := range peer.Tags { + if _, ok := srcMap[tagID.String()]; ok { + allowed = true + break + } + } + } + if allowed { + allowedPolicies = append(allowedPolicies, policy) + break + } + if _, ok := srcMap[tagID.String()]; ok { + if _, ok := dstMap["*"]; ok { + allowed = true + allowedPolicies = append(allowedPolicies, policy) + break + } + for tagID := range peer.Tags { + if _, ok := dstMap[tagID.String()]; ok { + allowed = true + break + } + } + } + if allowed { + allowedPolicies = append(allowedPolicies, policy) + break + } + } + for tagID := range peer.Tags { + allowed := false + if _, ok := dstMap[tagID.String()]; ok { + if _, ok := srcMap["*"]; ok { + allowed = true + allowedPolicies = append(allowedPolicies, policy) + break + } + for tagID := range node.Tags { + + if _, ok := srcMap[tagID.String()]; ok { + allowed = true + break + } + } + } + if allowed { + allowedPolicies = append(allowedPolicies, policy) + break + } + + if _, ok := srcMap[tagID.String()]; policy.AllowedDirection == models.TrafficDirectionBi && ok { + if _, ok := dstMap["*"]; ok { + allowed = true + allowedPolicies = append(allowedPolicies, policy) + break + } + for tagID := range node.Tags { + if _, ok := dstMap[tagID.String()]; ok { + allowed = true + break + } + } + } + if allowed { + allowedPolicies = append(allowedPolicies, policy) + break + } + } + } + + if len(allowedPolicies) > 0 { + return true, allowedPolicies + } + return false, allowedPolicies +} + // SortTagEntrys - Sorts slice of Tag entries by their id func SortAclEntrys(acls []models.Acl) { sort.Slice(acls, func(i, j int) bool { @@ -634,7 +805,9 @@ func CheckIfTagAsActivePolicy(tagID models.TagID, netID models.NetworkID) bool { } for _, dstTagI := range acl.Dst { if dstTagI.ID == models.DeviceAclID { - return true + if tagID.String() == dstTagI.Value { + return true + } } } } @@ -668,3 +841,227 @@ func RemoveDeviceTagFromAclPolicies(tagID models.TagID, netID models.NetworkID) } return nil } + +func getUserAclRulesForNode(targetnode *models.Node, + rules map[string]models.AclRule) map[string]models.AclRule { + userNodes := GetStaticUserNodesByNetwork(models.NetworkID(targetnode.Network)) + userGrpMap := GetUserGrpMap() + allowedUsers := make(map[string][]models.Acl) + acls := listUserPolicies(models.NetworkID(targetnode.Network)) + for nodeTag := range targetnode.Tags { + for _, acl := range acls { + if !acl.Enabled { + continue + } + dstTags := convAclTagToValueMap(acl.Dst) + if _, ok := dstTags[nodeTag.String()]; ok { + // get all src tags + for _, srcAcl := range acl.Src { + if srcAcl.ID == models.UserAclID { + allowedUsers[srcAcl.Value] = append(allowedUsers[srcAcl.Value], acl) + } else if srcAcl.ID == models.UserGroupAclID { + // fetch all users in the group + if usersMap, ok := userGrpMap[models.UserGroupID(srcAcl.Value)]; ok { + for userName := range usersMap { + allowedUsers[userName] = append(allowedUsers[userName], acl) + } + } + } + } + + } + } + } + for _, userNode := range userNodes { + if !userNode.StaticNode.Enabled { + continue + } + acls, ok := allowedUsers[userNode.StaticNode.OwnerID] + if !ok { + continue + } + for _, acl := range acls { + + if !acl.Enabled { + continue + } + + r := models.AclRule{ + ID: acl.ID, + AllowedProtocol: acl.Proto, + AllowedPorts: acl.Port, + Direction: acl.AllowedDirection, + Allowed: true, + } + // Get peers in the tags and add allowed rules + if userNode.StaticNode.Address != "" { + r.IPList = append(r.IPList, userNode.StaticNode.AddressIPNet4()) + } + if userNode.StaticNode.Address6 != "" { + r.IP6List = append(r.IP6List, userNode.StaticNode.AddressIPNet6()) + } + if aclRule, ok := rules[acl.ID]; ok { + aclRule.IPList = append(aclRule.IPList, r.IPList...) + aclRule.IP6List = append(aclRule.IP6List, r.IP6List...) + rules[acl.ID] = aclRule + } else { + rules[acl.ID] = r + } + } + } + return rules +} + +func GetAclRulesForNode(targetnode *models.Node) (rules map[string]models.AclRule) { + defer func() { + if !targetnode.IsIngressGateway { + rules = getUserAclRulesForNode(targetnode, rules) + } + + }() + rules = make(map[string]models.AclRule) + var taggedNodes map[models.TagID][]models.Node + if targetnode.IsIngressGateway { + taggedNodes = GetTagMapWithNodesByNetwork(models.NetworkID(targetnode.Network), false) + } else { + taggedNodes = GetTagMapWithNodesByNetwork(models.NetworkID(targetnode.Network), true) + } + + acls := listDevicePolicies(models.NetworkID(targetnode.Network)) + targetnode.Tags["*"] = struct{}{} + for nodeTag := range targetnode.Tags { + for _, acl := range acls { + if !acl.Enabled { + continue + } + srcTags := convAclTagToValueMap(acl.Src) + dstTags := convAclTagToValueMap(acl.Dst) + aclRule := models.AclRule{ + ID: acl.ID, + AllowedProtocol: acl.Proto, + AllowedPorts: acl.Port, + Direction: acl.AllowedDirection, + Allowed: true, + } + if acl.AllowedDirection == models.TrafficDirectionBi { + var existsInSrcTag bool + var existsInDstTag bool + + if _, ok := srcTags[nodeTag.String()]; ok { + existsInSrcTag = true + } + if _, ok := dstTags[nodeTag.String()]; ok { + existsInDstTag = true + } + + if existsInSrcTag && !existsInDstTag { + // get all dst tags + for dst := range dstTags { + if dst == nodeTag.String() { + continue + } + // Get peers in the tags and add allowed rules + nodes := taggedNodes[models.TagID(dst)] + for _, node := range nodes { + if node.ID == targetnode.ID { + continue + } + if node.Address.IP != nil { + aclRule.IPList = append(aclRule.IPList, node.AddressIPNet4()) + } + if node.Address6.IP != nil { + aclRule.IP6List = append(aclRule.IP6List, node.AddressIPNet6()) + } + if node.IsStatic && node.StaticNode.Address != "" { + aclRule.IPList = append(aclRule.IPList, node.StaticNode.AddressIPNet4()) + } + if node.IsStatic && node.StaticNode.Address6 != "" { + aclRule.IP6List = append(aclRule.IP6List, node.StaticNode.AddressIPNet6()) + } + } + } + } + if existsInDstTag && !existsInSrcTag { + // get all src tags + for src := range srcTags { + if src == nodeTag.String() { + continue + } + // Get peers in the tags and add allowed rules + nodes := taggedNodes[models.TagID(src)] + for _, node := range nodes { + if node.ID == targetnode.ID { + continue + } + if node.Address.IP != nil { + aclRule.IPList = append(aclRule.IPList, node.AddressIPNet4()) + } + if node.Address6.IP != nil { + aclRule.IP6List = append(aclRule.IP6List, node.AddressIPNet6()) + } + if node.IsStatic && node.StaticNode.Address != "" { + aclRule.IPList = append(aclRule.IPList, node.StaticNode.AddressIPNet4()) + } + if node.IsStatic && node.StaticNode.Address6 != "" { + aclRule.IP6List = append(aclRule.IP6List, node.StaticNode.AddressIPNet6()) + } + } + } + } + if existsInDstTag && existsInSrcTag { + nodes := taggedNodes[nodeTag] + for _, node := range nodes { + if node.ID == targetnode.ID { + continue + } + if node.Address.IP != nil { + aclRule.IPList = append(aclRule.IPList, node.AddressIPNet4()) + } + if node.Address6.IP != nil { + aclRule.IP6List = append(aclRule.IP6List, node.AddressIPNet6()) + } + if node.IsStatic && node.StaticNode.Address != "" { + aclRule.IPList = append(aclRule.IPList, node.StaticNode.AddressIPNet4()) + } + if node.IsStatic && node.StaticNode.Address6 != "" { + aclRule.IP6List = append(aclRule.IP6List, node.StaticNode.AddressIPNet6()) + } + } + } + } else { + _, all := dstTags["*"] + if _, ok := dstTags[nodeTag.String()]; ok || all { + // get all src tags + for src := range srcTags { + if src == nodeTag.String() { + continue + } + // Get peers in the tags and add allowed rules + nodes := taggedNodes[models.TagID(src)] + for _, node := range nodes { + if node.ID == targetnode.ID { + continue + } + if node.Address.IP != nil { + aclRule.IPList = append(aclRule.IPList, node.AddressIPNet4()) + } + if node.Address6.IP != nil { + aclRule.IP6List = append(aclRule.IP6List, node.AddressIPNet6()) + } + if node.IsStatic && node.StaticNode.Address != "" { + aclRule.IPList = append(aclRule.IPList, node.StaticNode.AddressIPNet4()) + } + if node.IsStatic && node.StaticNode.Address6 != "" { + aclRule.IP6List = append(aclRule.IP6List, node.StaticNode.AddressIPNet6()) + } + } + } + } + } + if len(aclRule.IPList) > 0 || len(aclRule.IP6List) > 0 { + rules[acl.ID] = aclRule + } + } + } + return rules +} diff --git a/logic/acls/nodeacls/retrieve.go b/logic/acls/nodeacls/retrieve.go index 4411c5b22..84895f44d 100644 --- a/logic/acls/nodeacls/retrieve.go +++ b/logic/acls/nodeacls/retrieve.go @@ -7,12 +7,16 @@ import ( "sync" "github.com/gravitl/netmaker/logic/acls" + "github.com/gravitl/netmaker/servercfg" ) var NodesAllowedACLMutex = &sync.Mutex{} // AreNodesAllowed - checks if nodes are allowed to communicate in their network ACL func AreNodesAllowed(networkID NetworkID, node1, node2 NodeID) bool { + if !servercfg.IsOldAclEnabled() { + return true + } NodesAllowedACLMutex.Lock() defer NodesAllowedACLMutex.Unlock() var currentNetworkACL, err = FetchAllACLs(networkID) diff --git a/logic/dns.go b/logic/dns.go index dd3450a21..b7bd3e895 100644 --- a/logic/dns.go +++ b/logic/dns.go @@ -2,6 +2,7 @@ package logic import ( "encoding/json" + "errors" "fmt" "os" "regexp" @@ -11,6 +12,7 @@ import ( "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/models" + "github.com/gravitl/netmaker/servercfg" "github.com/txn2/txeh" ) @@ -104,7 +106,7 @@ func GetNodeDNS(network string) ([]models.DNSEntry, error) { if err != nil { return dns, err } - + defaultDomain := servercfg.GetDefaultDomain() for _, node := range nodes { if node.Network != network { continue @@ -114,7 +116,7 @@ func GetNodeDNS(network string) ([]models.DNSEntry, error) { continue } var entry = models.DNSEntry{} - entry.Name = fmt.Sprintf("%s.%s", host.Name, network) + entry.Name = fmt.Sprintf("%s.%s.%s", host.Name, network, defaultDomain) entry.Network = network if node.Address.IP != nil { entry.Address = node.Address.IP.String() @@ -224,9 +226,17 @@ func SortDNSEntrys(unsortedDNSEntrys []models.DNSEntry) { }) } +// IsNetworkNameValid - checks if a netid of a network uses valid characters +func IsDNSEntryValid(d string) bool { + re := regexp.MustCompile(`^[A-Za-z0-9-.]+$`) + return re.MatchString(d) +} + // ValidateDNSCreate - checks if an entry is valid func ValidateDNSCreate(entry models.DNSEntry) error { - + if !IsDNSEntryValid(entry.Name) { + return errors.New("invalid input. Only uppercase letters (A-Z), lowercase letters (a-z), numbers (0-9), minus sign (-) and dots (.) are allowed") + } v := validator.New() _ = v.RegisterValidation("whitespace", func(f1 validator.FieldLevel) bool { diff --git a/logic/extpeers.go b/logic/extpeers.go index c03a0efa7..a005b54c5 100644 --- a/logic/extpeers.go +++ b/logic/extpeers.go @@ -456,6 +456,10 @@ func GetStaticNodeIps(node models.Node) (ips []net.IP) { func GetFwRulesOnIngressGateway(node models.Node) (rules []models.FwRule) { // fetch user access to static clients via policies + defer func() { + logger.Log(0, fmt.Sprintf("node.ID: %s, Rules: %+v\n", node.ID, rules)) + }() + defaultUserPolicy, _ := GetDefaultPolicy(models.NetworkID(node.Network), models.UserPolicy) defaultDevicePolicy, _ := GetDefaultPolicy(models.NetworkID(node.Network), models.DevicePolicy) nodes, _ := GetNetworkNodes(node.Network) @@ -468,36 +472,50 @@ func GetFwRulesOnIngressGateway(node models.Node) (rules []models.FwRule) { if peer.IsUserNode { continue } - if IsUserAllowedToCommunicate(userNodeI.StaticNode.OwnerID, peer) { + if ok, allowedPolicies := IsUserAllowedToCommunicate(userNodeI.StaticNode.OwnerID, peer); ok { if peer.IsStatic { if userNodeI.StaticNode.Address != "" { if !defaultUserPolicy.Enabled { - rules = append(rules, models.FwRule{ - SrcIP: userNodeI.StaticNode.AddressIPNet4(), - DstIP: peer.StaticNode.AddressIPNet4(), - Allow: true, - }) + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: userNodeI.StaticNode.AddressIPNet4(), + DstIP: peer.StaticNode.AddressIPNet4(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + rules = append(rules, models.FwRule{ + SrcIP: peer.StaticNode.AddressIPNet4(), + DstIP: userNodeI.StaticNode.AddressIPNet4(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + } } - rules = append(rules, models.FwRule{ - SrcIP: peer.StaticNode.AddressIPNet4(), - DstIP: userNodeI.StaticNode.AddressIPNet4(), - Allow: true, - }) + } if userNodeI.StaticNode.Address6 != "" { if !defaultUserPolicy.Enabled { - rules = append(rules, models.FwRule{ - SrcIP: userNodeI.StaticNode.AddressIPNet6(), - DstIP: peer.StaticNode.AddressIPNet6(), - Allow: true, - }) + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: userNodeI.StaticNode.AddressIPNet6(), + DstIP: peer.StaticNode.AddressIPNet6(), + Allow: true, + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + }) + rules = append(rules, models.FwRule{ + SrcIP: peer.StaticNode.AddressIPNet6(), + DstIP: userNodeI.StaticNode.AddressIPNet6(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + + } } - rules = append(rules, models.FwRule{ - SrcIP: peer.StaticNode.AddressIPNet6(), - DstIP: userNodeI.StaticNode.AddressIPNet6(), - Allow: true, - }) } if len(peer.StaticNode.ExtraAllowedIPs) > 0 { for _, additionalAllowedIPNet := range peer.StaticNode.ExtraAllowedIPs { @@ -526,29 +544,39 @@ func GetFwRulesOnIngressGateway(node models.Node) (rules []models.FwRule) { if userNodeI.StaticNode.Address != "" { if !defaultUserPolicy.Enabled { - rules = append(rules, models.FwRule{ - SrcIP: userNodeI.StaticNode.AddressIPNet4(), - DstIP: net.IPNet{ - IP: peer.Address.IP, - Mask: net.CIDRMask(32, 32), - }, - Allow: true, - }) + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: userNodeI.StaticNode.AddressIPNet4(), + DstIP: net.IPNet{ + IP: peer.Address.IP, + Mask: net.CIDRMask(32, 32), + }, + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + } + } } if userNodeI.StaticNode.Address6 != "" { - rules = append(rules, models.FwRule{ - SrcIP: userNodeI.StaticNode.AddressIPNet6(), - DstIP: net.IPNet{ - IP: peer.Address6.IP, - Mask: net.CIDRMask(128, 128), - }, - Allow: true, - }) + if !defaultUserPolicy.Enabled { + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: userNodeI.StaticNode.AddressIPNet6(), + DstIP: net.IPNet{ + IP: peer.Address6.IP, + Mask: net.CIDRMask(128, 128), + }, + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + } + } } } - } } } @@ -564,21 +592,48 @@ func GetFwRulesOnIngressGateway(node models.Node) (rules []models.FwRule) { if peer.StaticNode.ClientID == nodeI.StaticNode.ClientID || peer.IsUserNode { continue } - if IsNodeAllowedToCommunicate(nodeI, peer) { + if ok, allowedPolicies := IsNodeAllowedToCommunicate(nodeI, peer, true); ok { if peer.IsStatic { if nodeI.StaticNode.Address != "" { - rules = append(rules, models.FwRule{ - SrcIP: nodeI.StaticNode.AddressIPNet4(), - DstIP: peer.StaticNode.AddressIPNet4(), - Allow: true, - }) + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: nodeI.StaticNode.AddressIPNet4(), + DstIP: peer.StaticNode.AddressIPNet4(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + if policy.AllowedDirection == models.TrafficDirectionBi { + rules = append(rules, models.FwRule{ + SrcIP: peer.StaticNode.AddressIPNet4(), + DstIP: nodeI.StaticNode.AddressIPNet4(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + } + } + } if nodeI.StaticNode.Address6 != "" { - rules = append(rules, models.FwRule{ - SrcIP: nodeI.StaticNode.AddressIPNet6(), - DstIP: peer.StaticNode.AddressIPNet6(), - Allow: true, - }) + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: nodeI.StaticNode.AddressIPNet6(), + DstIP: peer.StaticNode.AddressIPNet6(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + if policy.AllowedDirection == models.TrafficDirectionBi { + rules = append(rules, models.FwRule{ + SrcIP: peer.StaticNode.AddressIPNet6(), + DstIP: nodeI.StaticNode.AddressIPNet6(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + } + } } if len(peer.StaticNode.ExtraAllowedIPs) > 0 { for _, additionalAllowedIPNet := range peer.StaticNode.ExtraAllowedIPs { @@ -605,24 +660,56 @@ func GetFwRulesOnIngressGateway(node models.Node) (rules []models.FwRule) { } } else { if nodeI.StaticNode.Address != "" { - rules = append(rules, models.FwRule{ - SrcIP: nodeI.StaticNode.AddressIPNet4(), - DstIP: net.IPNet{ - IP: peer.Address.IP, - Mask: net.CIDRMask(32, 32), - }, - Allow: true, - }) + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: nodeI.StaticNode.AddressIPNet4(), + DstIP: net.IPNet{ + IP: peer.Address.IP, + Mask: net.CIDRMask(32, 32), + }, + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + if policy.AllowedDirection == models.TrafficDirectionBi { + rules = append(rules, models.FwRule{ + SrcIP: net.IPNet{ + IP: peer.Address.IP, + Mask: net.CIDRMask(32, 32), + }, + DstIP: nodeI.StaticNode.AddressIPNet4(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + } + } } if nodeI.StaticNode.Address6 != "" { - rules = append(rules, models.FwRule{ - SrcIP: nodeI.StaticNode.AddressIPNet6(), - DstIP: net.IPNet{ - IP: peer.Address6.IP, - Mask: net.CIDRMask(128, 128), - }, - Allow: true, - }) + for _, policy := range allowedPolicies { + rules = append(rules, models.FwRule{ + SrcIP: nodeI.StaticNode.AddressIPNet6(), + DstIP: net.IPNet{ + IP: peer.Address6.IP, + Mask: net.CIDRMask(128, 128), + }, + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + if policy.AllowedDirection == models.TrafficDirectionBi { + rules = append(rules, models.FwRule{ + SrcIP: net.IPNet{ + IP: peer.Address6.IP, + Mask: net.CIDRMask(128, 128), + }, + DstIP: nodeI.StaticNode.AddressIPNet6(), + AllowedProtocol: policy.Proto, + AllowedPorts: policy.Port, + Allow: true, + }) + } + } } } @@ -650,11 +737,11 @@ func GetExtPeers(node, peer *models.Node) ([]wgtypes.PeerConfig, []models.IDandA continue } if extPeer.RemoteAccessClientID == "" { - if !IsNodeAllowedToCommunicate(extPeer.ConvertToStaticNode(), *peer) { + if ok := IsPeerAllowed(extPeer.ConvertToStaticNode(), *peer, true); !ok { continue } } else { - if !IsUserAllowedToCommunicate(extPeer.OwnerID, *peer) { + if ok, _ := IsUserAllowedToCommunicate(extPeer.OwnerID, *peer); !ok { continue } } @@ -739,7 +826,7 @@ func getExtpeerEgressRanges(node models.Node) (ranges, ranges6 []net.IPNet) { if len(extPeer.ExtraAllowedIPs) == 0 { continue } - if !IsNodeAllowedToCommunicate(extPeer.ConvertToStaticNode(), node) { + if ok, _ := IsNodeAllowedToCommunicate(extPeer.ConvertToStaticNode(), node, true); !ok { continue } for _, allowedRange := range extPeer.ExtraAllowedIPs { @@ -766,7 +853,7 @@ func getExtpeersExtraRoutes(node models.Node) (egressRoutes []models.EgressNetwo if len(extPeer.ExtraAllowedIPs) == 0 { continue } - if !IsNodeAllowedToCommunicate(extPeer.ConvertToStaticNode(), node) { + if ok, _ := IsNodeAllowedToCommunicate(extPeer.ConvertToStaticNode(), node, true); !ok { continue } egressRoutes = append(egressRoutes, getExtPeerEgressRoute(node, extPeer)...) diff --git a/logic/nodes.go b/logic/nodes.go index 34eebe2e4..368d7d674 100644 --- a/logic/nodes.go +++ b/logic/nodes.go @@ -5,7 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "maps" "net" + "slices" "sort" "sync" "time" @@ -24,8 +26,10 @@ import ( ) var ( - nodeCacheMutex = &sync.RWMutex{} - nodesCacheMap = make(map[string]models.Node) + nodeCacheMutex = &sync.RWMutex{} + nodeNetworkCacheMutex = &sync.RWMutex{} + nodesCacheMap = make(map[string]models.Node) + nodesNetworkCacheMap = make(map[string]map[string]models.Node) ) func getNodeFromCache(nodeID string) (node models.Node, ok bool) { @@ -48,12 +52,37 @@ func deleteNodeFromCache(nodeID string) { delete(nodesCacheMap, nodeID) nodeCacheMutex.Unlock() } +func deleteNodeFromNetworkCache(nodeID string, network string) { + nodeNetworkCacheMutex.Lock() + delete(nodesNetworkCacheMap[network], nodeID) + nodeNetworkCacheMutex.Unlock() +} + +func storeNodeInNetworkCache(node models.Node, network string) { + nodeNetworkCacheMutex.Lock() + if nodesNetworkCacheMap[network] == nil { + nodesNetworkCacheMap[network] = make(map[string]models.Node) + } + nodesNetworkCacheMap[network][node.ID.String()] = node + nodeNetworkCacheMutex.Unlock() +} func storeNodeInCache(node models.Node) { nodeCacheMutex.Lock() nodesCacheMap[node.ID.String()] = node nodeCacheMutex.Unlock() } +func loadNodesIntoNetworkCache(nMap map[string]models.Node) { + nodeNetworkCacheMutex.Lock() + for _, v := range nMap { + network := v.Network + if nodesNetworkCacheMap[network] == nil { + nodesNetworkCacheMap[network] = make(map[string]models.Node) + } + nodesNetworkCacheMap[network][v.ID.String()] = v + } + nodeNetworkCacheMutex.Unlock() +} func loadNodesIntoCache(nMap map[string]models.Node) { nodeCacheMutex.Lock() @@ -63,6 +92,7 @@ func loadNodesIntoCache(nMap map[string]models.Node) { func ClearNodeCache() { nodeCacheMutex.Lock() nodesCacheMap = make(map[string]models.Node) + nodesNetworkCacheMap = make(map[string]map[string]models.Node) nodeCacheMutex.Unlock() } @@ -77,6 +107,12 @@ const ( // GetNetworkNodes - gets the nodes of a network func GetNetworkNodes(network string) ([]models.Node, error) { + + if networkNodes, ok := nodesNetworkCacheMap[network]; ok { + nodeNetworkCacheMutex.Lock() + defer nodeNetworkCacheMutex.Unlock() + return slices.Collect(maps.Values(networkNodes)), nil + } allnodes, err := GetAllNodes() if err != nil { return []models.Node{}, err @@ -99,6 +135,12 @@ func GetHostNodes(host *models.Host) []models.Node { // GetNetworkNodesMemory - gets all nodes belonging to a network from list in memory func GetNetworkNodesMemory(allNodes []models.Node, network string) []models.Node { + + if networkNodes, ok := nodesNetworkCacheMap[network]; ok { + nodeNetworkCacheMutex.Lock() + defer nodeNetworkCacheMutex.Unlock() + return slices.Collect(maps.Values(networkNodes)) + } var nodes = []models.Node{} for i := range allNodes { node := allNodes[i] @@ -123,6 +165,7 @@ func UpdateNodeCheckin(node *models.Node) error { } if servercfg.CacheEnabled() { storeNodeInCache(*node) + storeNodeInNetworkCache(*node, node.Network) } return nil } @@ -140,6 +183,7 @@ func UpsertNode(newNode *models.Node) error { } if servercfg.CacheEnabled() { storeNodeInCache(*newNode) + storeNodeInNetworkCache(*newNode, newNode.Network) } return nil } @@ -179,6 +223,17 @@ func UpdateNode(currentNode *models.Node, newNode *models.Node) error { } if servercfg.CacheEnabled() { storeNodeInCache(*newNode) + storeNodeInNetworkCache(*newNode, newNode.Network) + if _, ok := allocatedIpMap[newNode.Network]; ok { + if newNode.Address.IP != nil && !newNode.Address.IP.Equal(currentNode.Address.IP) { + AddIpToAllocatedIpMap(newNode.Network, newNode.Address.IP) + RemoveIpFromAllocatedIpMap(currentNode.Network, currentNode.Address.IP.String()) + } + if newNode.Address6.IP != nil && !newNode.Address6.IP.Equal(currentNode.Address6.IP) { + AddIpToAllocatedIpMap(newNode.Network, newNode.Address6.IP) + RemoveIpFromAllocatedIpMap(currentNode.Network, currentNode.Address6.IP.String()) + } + } } return nil } @@ -288,6 +343,7 @@ func DeleteNodeByID(node *models.Node) error { } if servercfg.CacheEnabled() { deleteNodeFromCache(node.ID.String()) + deleteNodeFromNetworkCache(node.ID.String(), node.Network) } if servercfg.IsDNSMode() { SetDNS() @@ -350,6 +406,7 @@ func GetAllNodes() ([]models.Node, error) { nodesMap := make(map[string]models.Node) if servercfg.CacheEnabled() { defer loadNodesIntoCache(nodesMap) + defer loadNodesIntoNetworkCache(nodesMap) } collection, err := database.FetchRecords(database.NODES_TABLE_NAME) if err != nil { @@ -388,6 +445,20 @@ func AddStaticNodestoList(nodes []models.Node) []models.Node { return nodes } +func AddStatusToNodes(nodes []models.Node) (nodesWithStatus []models.Node) { + aclDefaultPolicyStatusMap := make(map[string]bool) + for _, node := range nodes { + if _, ok := aclDefaultPolicyStatusMap[node.Network]; !ok { + // check default policy if all allowed return true + defaultPolicy, _ := GetDefaultPolicy(models.NetworkID(node.Network), models.DevicePolicy) + aclDefaultPolicyStatusMap[node.Network] = defaultPolicy.Enabled + } + GetNodeStatus(&node, aclDefaultPolicyStatusMap[node.Network]) + nodesWithStatus = append(nodesWithStatus, node) + } + return +} + // GetNetworkByNode - gets the network model from a node func GetNetworkByNode(node *models.Node) (models.Network, error) { @@ -459,6 +530,7 @@ func GetNodeByID(uuid string) (models.Node, error) { } if servercfg.CacheEnabled() { storeNodeInCache(node) + storeNodeInNetworkCache(node, node.Network) } return node, nil } @@ -612,6 +684,7 @@ func createNode(node *models.Node) error { } if servercfg.CacheEnabled() { storeNodeInCache(*node) + storeNodeInNetworkCache(*node, node.Network) } if _, ok := allocatedIpMap[node.Network]; ok { if node.Address.IP != nil { @@ -663,6 +736,26 @@ func ValidateParams(nodeid, netid string) (models.Node, error) { return node, nil } +func ValidateNodeIp(currentNode *models.Node, newNode *models.ApiNode) error { + + if currentNode.Address.IP != nil && currentNode.Address.String() != newNode.Address { + newIp, _, _ := net.ParseCIDR(newNode.Address) + ipAllocated := allocatedIpMap[currentNode.Network] + if _, ok := ipAllocated[newIp.String()]; ok { + return errors.New("ip specified is already allocated: " + newNode.Address) + } + } + if currentNode.Address6.IP != nil && currentNode.Address6.String() != newNode.Address6 { + newIp, _, _ := net.ParseCIDR(newNode.Address6) + ipAllocated := allocatedIpMap[currentNode.Network] + if _, ok := ipAllocated[newIp.String()]; ok { + return errors.New("ip specified is already allocated: " + newNode.Address6) + } + } + + return nil +} + func ValidateEgressRange(gateway models.EgressGatewayRequest) error { network, err := GetNetworkSettings(gateway.NetID) if err != nil { @@ -725,7 +818,7 @@ func GetTagMapWithNodes() (tagNodesMap map[models.TagID][]models.Node) { return } -func GetTagMapWithNodesByNetwork(netID models.NetworkID) (tagNodesMap map[models.TagID][]models.Node) { +func GetTagMapWithNodesByNetwork(netID models.NetworkID, withStaticNodes bool) (tagNodesMap map[models.TagID][]models.Node) { tagNodesMap = make(map[models.TagID][]models.Node) nodes, _ := GetNetworkNodes(netID.String()) for _, nodeI := range nodes { @@ -736,6 +829,10 @@ func GetTagMapWithNodesByNetwork(netID models.NetworkID) (tagNodesMap map[models tagNodesMap[nodeTagID] = append(tagNodesMap[nodeTagID], nodeI) } } + tagNodesMap["*"] = nodes + if !withStaticNodes { + return + } return AddTagMapWithStaticNodes(netID, tagNodesMap) } @@ -749,6 +846,31 @@ func AddTagMapWithStaticNodes(netID models.NetworkID, if extclient.Tags == nil || extclient.RemoteAccessClientID != "" { continue } + for tagID := range extclient.Tags { + tagNodesMap[tagID] = append(tagNodesMap[tagID], models.Node{ + IsStatic: true, + StaticNode: extclient, + }) + tagNodesMap["*"] = append(tagNodesMap["*"], models.Node{ + IsStatic: true, + StaticNode: extclient, + }) + } + + } + return tagNodesMap +} + +func AddTagMapWithStaticNodesWithUsers(netID models.NetworkID, + tagNodesMap map[models.TagID][]models.Node) map[models.TagID][]models.Node { + extclients, err := GetNetworkExtClients(netID.String()) + if err != nil { + return tagNodesMap + } + for _, extclient := range extclients { + if extclient.Tags == nil { + continue + } for tagID := range extclient.Tags { tagNodesMap[tagID] = append(tagNodesMap[tagID], models.Node{ IsStatic: true, diff --git a/logic/peers.go b/logic/peers.go index f67162636..b665e51ff 100644 --- a/logic/peers.go +++ b/logic/peers.go @@ -74,8 +74,10 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N ServerVersion: servercfg.GetVersion(), ServerAddrs: []models.ServerAddr{}, FwUpdate: models.FwUpdate{ + AllowAll: true, EgressInfo: make(map[string]models.EgressInfo), IngressInfo: make(map[string]models.IngressInfo), + AclRules: make(map[string]models.AclRule), }, PeerIDs: make(models.PeerMap, 0), Peers: []wgtypes.PeerConfig{}, @@ -83,6 +85,24 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N HostNetworkInfo: models.HostInfoMap{}, EndpointDetection: servercfg.IsEndpointDetectionEnabled(), } + defer func() { + if !hostPeerUpdate.FwUpdate.AllowAll { + aclRule := models.AclRule{ + ID: "allowed-network-rules", + AllowedProtocol: models.ALL, + Direction: models.TrafficDirectionBi, + Allowed: true, + } + for _, allowedNet := range hostPeerUpdate.FwUpdate.AllowedNetworks { + if allowedNet.IP.To4() != nil { + aclRule.IPList = append(aclRule.IPList, allowedNet) + } else { + aclRule.IP6List = append(aclRule.IP6List, allowedNet) + } + } + hostPeerUpdate.FwUpdate.AclRules["allowed-network-rules"] = aclRule + } + }() slog.Debug("peer update for host", "hostId", host.ID.String()) peerIndexMap := make(map[string]int) @@ -154,6 +174,22 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N if !hostPeerUpdate.IsInternetGw { hostPeerUpdate.IsInternetGw = IsInternetGw(node) } + defaultUserPolicy, _ := GetDefaultPolicy(models.NetworkID(node.Network), models.UserPolicy) + defaultDevicePolicy, _ := GetDefaultPolicy(models.NetworkID(node.Network), models.DevicePolicy) + + if defaultDevicePolicy.Enabled && defaultUserPolicy.Enabled { + if node.NetworkRange.IP != nil { + hostPeerUpdate.FwUpdate.AllowedNetworks = append(hostPeerUpdate.FwUpdate.AllowedNetworks, node.NetworkRange) + } + if node.NetworkRange6.IP != nil { + hostPeerUpdate.FwUpdate.AllowedNetworks = append(hostPeerUpdate.FwUpdate.AllowedNetworks, node.NetworkRange6) + } + + } else { + hostPeerUpdate.FwUpdate.AllowAll = false + hostPeerUpdate.FwUpdate.AclRules = GetAclRulesForNode(&node) + } + currentPeers := GetNetworkNodesMemory(allNodes, node.Network) for _, peer := range currentPeers { peer := peer @@ -255,11 +291,12 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N peerConfig.Endpoint.Port = peerHost.ListenPort } allowedips := GetAllowedIPs(&node, &peer, nil) + allowedToComm := IsPeerAllowed(node, peer, false) if peer.Action != models.NODE_DELETE && !peer.PendingDelete && peer.Connected && nodeacls.AreNodesAllowed(nodeacls.NetworkID(node.Network), nodeacls.NodeID(node.ID.String()), nodeacls.NodeID(peer.ID.String())) && - IsNodeAllowedToCommunicate(node, peer) && + (defaultDevicePolicy.Enabled || allowedToComm) && (deletedNode == nil || (deletedNode != nil && peer.ID.String() != deletedNode.ID.String())) { peerConfig.AllowedIPs = allowedips // only append allowed IPs if valid connection } @@ -309,8 +346,6 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N hostPeerUpdate.FwUpdate.IsIngressGw = true extPeers, extPeerIDAndAddrs, egressRoutes, err = GetExtPeers(&node, &node) if err == nil { - defaultUserPolicy, _ := GetDefaultPolicy(models.NetworkID(node.Network), models.UserPolicy) - defaultDevicePolicy, _ := GetDefaultPolicy(models.NetworkID(node.Network), models.DevicePolicy) if !defaultDevicePolicy.Enabled || !defaultUserPolicy.Enabled { ingFwUpdate := models.IngressInfo{ IngressID: node.ID.String(), @@ -426,6 +461,8 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N } hostPeerUpdate.ManageDNS = servercfg.GetManageDNS() + hostPeerUpdate.Stun = servercfg.IsStunEnabled() + hostPeerUpdate.StunServers = servercfg.GetStunServers() return hostPeerUpdate, nil } diff --git a/logic/proc.go b/logic/proc.go index fec258d75..fadc69316 100644 --- a/logic/proc.go +++ b/logic/proc.go @@ -2,6 +2,7 @@ package logic import ( "os" + "runtime" "runtime/pprof" "github.com/gravitl/netmaker/logger" @@ -22,3 +23,15 @@ func StopCPUProfiling(f *os.File) { pprof.StopCPUProfile() f.Close() } + +func StartMemProfiling() { + f, err := os.OpenFile("/root/data/mem.prof", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0755) + if err != nil { + logger.Log(0, "could not create Memory profile: ", err.Error()) + } + defer f.Close() + runtime.GC() // get up-to-date statistics + if err = pprof.WriteHeapProfile(f); err != nil { + logger.Log(0, "could not write memory profile: ", err.Error()) + } +} diff --git a/logic/status.go b/logic/status.go new file mode 100644 index 000000000..e0511ae05 --- /dev/null +++ b/logic/status.go @@ -0,0 +1,26 @@ +package logic + +import ( + "time" + + "github.com/gravitl/netmaker/models" +) + +var GetNodeStatus = getNodeStatus + +func getNodeStatus(node *models.Node, t bool) { + // On CE check only last check-in time + if node.IsStatic { + if !node.StaticNode.Enabled { + node.Status = models.OfflineSt + return + } + node.Status = models.OnlineSt + return + } + if time.Since(node.LastCheckIn) > time.Minute*10 { + node.Status = models.OfflineSt + return + } + node.Status = models.OnlineSt +} diff --git a/logic/tags.go b/logic/tags.go index 7cdf0f324..0c2aa095e 100644 --- a/logic/tags.go +++ b/logic/tags.go @@ -85,7 +85,7 @@ func ListTagsWithNodes(netID models.NetworkID) ([]models.TagListResp, error) { if err != nil { return []models.TagListResp{}, err } - tagsNodeMap := GetTagMapWithNodesByNetwork(netID) + tagsNodeMap := GetTagMapWithNodesByNetwork(netID, true) resp := []models.TagListResp{} for _, tagI := range tags { tagRespI := models.TagListResp{ diff --git a/logic/user_mgmt.go b/logic/user_mgmt.go index 56395c78c..f6eccac20 100644 --- a/logic/user_mgmt.go +++ b/logic/user_mgmt.go @@ -98,6 +98,25 @@ func ListPlatformRoles() ([]models.UserRolePermissionTemplate, error) { return userRoles, nil } +func GetUserGrpMap() map[models.UserGroupID]map[string]struct{} { + grpUsersMap := make(map[models.UserGroupID]map[string]struct{}) + users, _ := GetUsersDB() + for _, user := range users { + for gID := range user.UserGroups { + if grpUsers, ok := grpUsersMap[gID]; ok { + grpUsers[user.UserName] = struct{}{} + grpUsersMap[gID] = grpUsers + } else { + grpUsersMap[gID] = make(map[string]struct{}) + grpUsersMap[gID][user.UserName] = struct{}{} + } + } + + } + + return grpUsersMap +} + func userRolesInit() { d, _ := json.Marshal(SuperAdminPermissionTemplate) database.Insert(SuperAdminPermissionTemplate.ID.String(), string(d), database.USER_PERMISSIONS_TABLE_NAME) diff --git a/logic/util.go b/logic/util.go index 3b410c586..bff2ced67 100644 --- a/logic/util.go +++ b/logic/util.go @@ -6,11 +6,14 @@ import ( "encoding/base32" "encoding/base64" "encoding/json" + "fmt" "net" "os" "strings" "time" + "unicode" + "github.com/blang/semver" "github.com/c-robinson/iplib" "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logger" @@ -148,4 +151,28 @@ func IsSlicesEqual(a, b []string) bool { return true } -// == private == +// VersionLessThan checks if v1 < v2 semantically +// dev is the latest version +func VersionLessThan(v1, v2 string) (bool, error) { + if v1 == "dev" { + return false, nil + } + if v2 == "dev" { + return true, nil + } + semVer1 := strings.TrimFunc(v1, func(r rune) bool { + return !unicode.IsNumber(r) + }) + semVer2 := strings.TrimFunc(v2, func(r rune) bool { + return !unicode.IsNumber(r) + }) + sv1, err := semver.Parse(semVer1) + if err != nil { + return false, fmt.Errorf("failed to parse semver1 (%s): %w", semVer1, err) + } + sv2, err := semver.Parse(semVer2) + if err != nil { + return false, fmt.Errorf("failed to parse semver2 (%s): %w", semVer2, err) + } + return sv1.LT(sv2), nil +} diff --git a/main.go b/main.go index 10bb52b8b..fd46ffcf8 100644 --- a/main.go +++ b/main.go @@ -24,13 +24,14 @@ import ( "github.com/gravitl/netmaker/netclient/ncutils" "github.com/gravitl/netmaker/servercfg" "github.com/gravitl/netmaker/serverctl" + _ "go.uber.org/automaxprocs" "golang.org/x/exp/slog" ) -var version = "v0.26.0" +var version = "v0.30.0" // @title NetMaker -// @version 0.26.0 +// @version 0.30.0 // @description NetMaker API Docs // @tag.name APIUsage // @tag.description.markdown @@ -99,6 +100,15 @@ func initialize() { // Client Mode Prereq Check logger.FatalLog("Error connecting to database: ", err.Error()) } logger.Log(0, "database successfully connected") + + //initialize cache + _, _ = logic.GetNetworks() + _, _ = logic.GetAllNodes() + _, _ = logic.GetAllHosts() + _, _ = logic.GetAllExtClients() + _ = logic.ListAcls() + _, _ = logic.GetAllEnrollmentKeys() + migrate.Run() logic.SetJWTSecret() diff --git a/main_ee.go b/main_ee.go index 90001d870..3beb9e189 100644 --- a/main_ee.go +++ b/main_ee.go @@ -3,7 +3,10 @@ package main -import "github.com/gravitl/netmaker/pro" +import ( + "github.com/gravitl/netmaker/pro" + _ "go.uber.org/automaxprocs" +) func init() { pro.InitPro() diff --git a/migrate/migrate.go b/migrate/migrate.go index b4a866ab7..824deecae 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -20,8 +20,6 @@ import ( // Run - runs all migrations func Run() { - _, _ = logic.GetAllNodes() - _, _ = logic.GetAllHosts() updateEnrollmentKeys() assignSuperAdmin() createDefaultTagsAndPolicies() @@ -439,5 +437,7 @@ func createDefaultTagsAndPolicies() { for _, network := range networks { logic.CreateDefaultTags(models.NetworkID(network.NetID)) logic.CreateDefaultAclNetworkPolicies(models.NetworkID(network.NetID)) + } + logic.MigrateAclPolicies() } diff --git a/models/acl.go b/models/acl.go index d8c302ca9..9f303fd1d 100644 --- a/models/acl.go +++ b/models/acl.go @@ -1,6 +1,7 @@ package models import ( + "net" "time" ) @@ -14,6 +15,32 @@ const ( TrafficDirectionBi ) +// Protocol - allowed protocol +type Protocol string + +const ( + ALL Protocol = "all" + UDP Protocol = "udp" + TCP Protocol = "tcp" + ICMP Protocol = "icmp" +) + +type ServiceType string + +const ( + Http = "HTTP" + Https = "HTTPS" + AllTCP = "All TCP" + AllUDP = "All UDP" + ICMPService = "ICMP" + Custom = "Custom" + Any = "Any" +) + +func (p Protocol) String() string { + return string(p) +} + type AclPolicyType string const ( @@ -59,6 +86,9 @@ type Acl struct { RuleType AclPolicyType `json:"policy_type"` Src []AclPolicyTag `json:"src_type"` Dst []AclPolicyTag `json:"dst_type"` + Proto Protocol `json:"protocol"` // tcp, udp, etc. + ServiceType string `json:"type"` + Port []string `json:"ports"` AllowedDirection AllowedTrafficDirection `json:"allowed_traffic_direction"` Enabled bool `json:"enabled"` CreatedBy string `json:"created_by"` @@ -66,7 +96,25 @@ type Acl struct { } type AclPolicyTypes struct { + ProtocolTypes []ProtocolType RuleTypes []AclPolicyType `json:"policy_types"` SrcGroupTypes []AclGroupType `json:"src_grp_types"` DstGroupTypes []AclGroupType `json:"dst_grp_types"` } + +type ProtocolType struct { + Name string `json:"name"` + AllowedProtocols []Protocol `json:"allowed_protocols"` + PortRange string `json:"port_range"` + AllowPortSetting bool `json:"allow_port_setting"` +} + +type AclRule struct { + ID string `json:"id"` + IPList []net.IPNet `json:"ip_list"` + IP6List []net.IPNet `json:"ip6_list"` + AllowedProtocol Protocol `json:"allowed_protocols"` // tcp, udp, etc. + AllowedPorts []string `json:"allowed_ports"` + Direction AllowedTrafficDirection `json:"direction"` // single or two-way + Allowed bool +} diff --git a/models/api_node.go b/models/api_node.go index 30e08c639..995c4f581 100644 --- a/models/api_node.go +++ b/models/api_node.go @@ -16,10 +16,10 @@ type ApiNode struct { Address6 string `json:"address6" validate:"omitempty,cidrv6"` LocalAddress string `json:"localaddress" validate:"omitempty,cidr"` AllowedIPs []string `json:"allowedips"` - LastModified int64 `json:"lastmodified"` - ExpirationDateTime int64 `json:"expdatetime"` - LastCheckIn int64 `json:"lastcheckin"` - LastPeerUpdate int64 `json:"lastpeerupdate"` + LastModified int64 `json:"lastmodified" swaggertype:"primitive,integer" format:"int64"` + ExpirationDateTime int64 `json:"expdatetime" swaggertype:"primitive,integer" format:"int64"` + LastCheckIn int64 `json:"lastcheckin" swaggertype:"primitive,integer" format:"int64"` + LastPeerUpdate int64 `json:"lastpeerupdate" swaggertype:"primitive,integer" format:"int64"` Network string `json:"network"` NetworkRange string `json:"networkrange"` NetworkRange6 string `json:"networkrange6"` @@ -52,6 +52,7 @@ type ApiNode struct { IsStatic bool `json:"is_static"` IsUserNode bool `json:"is_user_node"` StaticNode ExtClient `json:"static_node"` + Status NodeStatus `json:"status"` } // ApiNode.ConvertToServerNode - converts an api node to a server node @@ -192,6 +193,7 @@ func (nm *Node) ConvertToAPINode() *ApiNode { apiNode.IsStatic = nm.IsStatic apiNode.IsUserNode = nm.IsUserNode apiNode.StaticNode = nm.StaticNode + apiNode.Status = nm.Status return &apiNode } diff --git a/models/enrollment_key.go b/models/enrollment_key.go index f133d7558..7ed6e56a8 100644 --- a/models/enrollment_key.go +++ b/models/enrollment_key.go @@ -58,7 +58,7 @@ type EnrollmentKey struct { // APIEnrollmentKey - used to create enrollment keys via API type APIEnrollmentKey struct { - Expiration int64 `json:"expiration"` + Expiration int64 `json:"expiration" swaggertype:"primitive,integer" format:"int64"` UsesRemaining int `json:"uses_remaining"` Networks []string `json:"networks"` Unlimited bool `json:"unlimited"` diff --git a/models/extclient.go b/models/extclient.go index a6214c34a..e9d3708b8 100644 --- a/models/extclient.go +++ b/models/extclient.go @@ -13,7 +13,7 @@ type ExtClient struct { AllowedIPs []string `json:"allowed_ips"` IngressGatewayID string `json:"ingressgatewayid" bson:"ingressgatewayid"` IngressGatewayEndpoint string `json:"ingressgatewayendpoint" bson:"ingressgatewayendpoint"` - LastModified int64 `json:"lastmodified" bson:"lastmodified"` + LastModified int64 `json:"lastmodified" bson:"lastmodified" swaggertype:"primitive,integer" format:"int64"` Enabled bool `json:"enabled" bson:"enabled"` OwnerID string `json:"ownerid" bson:"ownerid"` DeniedACLs map[string]struct{} `json:"deniednodeacls" bson:"acls,omitempty"` @@ -21,20 +21,29 @@ type ExtClient struct { PostUp string `json:"postup" bson:"postup"` PostDown string `json:"postdown" bson:"postdown"` Tags map[TagID]struct{} `json:"tags"` + Os string `json:"os"` + DeviceName string `json:"device_name"` + PublicEndpoint string `json:"public_endpoint"` + Country string `json:"country"` } // CustomExtClient - struct for CustomExtClient params type CustomExtClient struct { - ClientID string `json:"clientid,omitempty"` - PublicKey string `json:"publickey,omitempty"` - DNS string `json:"dns,omitempty"` - ExtraAllowedIPs []string `json:"extraallowedips,omitempty"` - Enabled bool `json:"enabled,omitempty"` - DeniedACLs map[string]struct{} `json:"deniednodeacls" bson:"acls,omitempty"` - RemoteAccessClientID string `json:"remote_access_client_id"` // unique ID (MAC address) of RAC machine - PostUp string `json:"postup" bson:"postup" validate:"max=1024"` - PostDown string `json:"postdown" bson:"postdown" validate:"max=1024"` - Tags map[TagID]struct{} `json:"tags"` + ClientID string `json:"clientid,omitempty"` + PublicKey string `json:"publickey,omitempty"` + DNS string `json:"dns,omitempty"` + ExtraAllowedIPs []string `json:"extraallowedips,omitempty"` + Enabled bool `json:"enabled,omitempty"` + DeniedACLs map[string]struct{} `json:"deniednodeacls" bson:"acls,omitempty"` + RemoteAccessClientID string `json:"remote_access_client_id"` // unique ID (MAC address) of RAC machine + PostUp string `json:"postup" bson:"postup" validate:"max=1024"` + PostDown string `json:"postdown" bson:"postdown" validate:"max=1024"` + Tags map[TagID]struct{} `json:"tags"` + Os string `json:"os"` + DeviceName string `json:"device_name"` + IsAlreadyConnectedToInetGw bool `json:"is_already_connected_to_inet_gw"` + PublicEndpoint string `json:"public_endpoint"` + Country string `json:"country"` } func (ext *ExtClient) ConvertToStaticNode() Node { diff --git a/models/host.go b/models/host.go index 2781dee0e..c6d5eaa3c 100644 --- a/models/host.go +++ b/models/host.go @@ -71,7 +71,7 @@ type Host struct { IsDefault bool `json:"isdefault" yaml:"isdefault"` NatType string `json:"nat_type,omitempty" yaml:"nat_type,omitempty"` TurnEndpoint *netip.AddrPort `json:"turn_endpoint,omitempty" yaml:"turn_endpoint,omitempty"` - PersistentKeepalive time.Duration `json:"persistentkeepalive" yaml:"persistentkeepalive"` + PersistentKeepalive time.Duration `json:"persistentkeepalive" swaggertype:"primitive,integer" format:"int64" yaml:"persistentkeepalive"` } // FormatBool converts a boolean to a [yes|no] string diff --git a/models/metrics.go b/models/metrics.go index 459c7f17c..2e70a1780 100644 --- a/models/metrics.go +++ b/models/metrics.go @@ -15,14 +15,14 @@ type Metrics struct { // Metric - holds a metric for data between nodes type Metric struct { NodeName string `json:"node_name" bson:"node_name" yaml:"node_name"` - Uptime int64 `json:"uptime" bson:"uptime" yaml:"uptime"` - TotalTime int64 `json:"totaltime" bson:"totaltime" yaml:"totaltime"` - Latency int64 `json:"latency" bson:"latency" yaml:"latency"` - TotalReceived int64 `json:"totalreceived" bson:"totalreceived" yaml:"totalreceived"` - LastTotalReceived int64 `json:"lasttotalreceived" bson:"lasttotalreceived" yaml:"lasttotalreceived"` - TotalSent int64 `json:"totalsent" bson:"totalsent" yaml:"totalsent"` - LastTotalSent int64 `json:"lasttotalsent" bson:"lasttotalsent" yaml:"lasttotalsent"` - ActualUptime time.Duration `json:"actualuptime" bson:"actualuptime" yaml:"actualuptime"` + Uptime int64 `json:"uptime" bson:"uptime" yaml:"uptime" swaggertype:"primitive,integer" format:"int64"` + TotalTime int64 `json:"totaltime" bson:"totaltime" yaml:"totaltime" swaggertype:"primitive,integer" format:"int64"` + Latency int64 `json:"latency" bson:"latency" yaml:"latency" swaggertype:"primitive,integer" format:"int64"` + TotalReceived int64 `json:"totalreceived" bson:"totalreceived" yaml:"totalreceived" swaggertype:"primitive,integer" format:"int64"` + LastTotalReceived int64 `json:"lasttotalreceived" bson:"lasttotalreceived" yaml:"lasttotalreceived" swaggertype:"primitive,integer" format:"int64"` + TotalSent int64 `json:"totalsent" bson:"totalsent" yaml:"totalsent" swaggertype:"primitive,integer" format:"int64"` + LastTotalSent int64 `json:"lasttotalsent" bson:"lasttotalsent" yaml:"lasttotalsent" swaggertype:"primitive,integer" format:"int64"` + ActualUptime time.Duration `json:"actualuptime" swaggertype:"primitive,integer" format:"int64" bson:"actualuptime" yaml:"actualuptime"` PercentUp float64 `json:"percentup" bson:"percentup" yaml:"percentup"` Connected bool `json:"connected" bson:"connected" yaml:"connected"` } diff --git a/models/mqtt.go b/models/mqtt.go index f0048a014..c5921f381 100644 --- a/models/mqtt.go +++ b/models/mqtt.go @@ -25,12 +25,16 @@ type HostPeerUpdate struct { ReplacePeers bool `json:"replace_peers"` EndpointDetection bool `json:"endpoint_detection"` ManageDNS bool `yaml:"manage_dns"` + Stun bool `yaml:"stun"` + StunServers string `yaml:"stun_servers"` } type FwRule struct { - SrcIP net.IPNet - DstIP net.IPNet - Allow bool + SrcIP net.IPNet `json:"src_ip"` + DstIP net.IPNet `json:"dst_ip"` + AllowedProtocol Protocol `json:"allowed_protocols"` // tcp, udp, etc. + AllowedPorts []string `json:"allowed_ports"` + Allow bool `json:"allow"` } // IngressInfo - struct for ingress info @@ -90,10 +94,13 @@ type KeyUpdate struct { // FwUpdate - struct for firewall updates type FwUpdate struct { - IsEgressGw bool `json:"is_egress_gw"` - IsIngressGw bool `json:"is_ingress_gw"` - EgressInfo map[string]EgressInfo `json:"egress_info"` - IngressInfo map[string]IngressInfo `json:"ingress_info"` + AllowAll bool `json:"allow_all"` + AllowedNetworks []net.IPNet `json:"networks"` + IsEgressGw bool `json:"is_egress_gw"` + IsIngressGw bool `json:"is_ingress_gw"` + EgressInfo map[string]EgressInfo `json:"egress_info"` + IngressInfo map[string]IngressInfo `json:"ingress_info"` + AclRules map[string]AclRule `json:"acl_rules"` } // FailOverMeReq - struct for failover req diff --git a/models/network.go b/models/network.go index 32d95f865..dbb2fc4a4 100644 --- a/models/network.go +++ b/models/network.go @@ -11,8 +11,8 @@ type Network struct { AddressRange string `json:"addressrange" bson:"addressrange" validate:"omitempty,cidrv4"` AddressRange6 string `json:"addressrange6" bson:"addressrange6" validate:"omitempty,cidrv6"` NetID string `json:"netid" bson:"netid" validate:"required,min=1,max=32,netid_valid"` - NodesLastModified int64 `json:"nodeslastmodified" bson:"nodeslastmodified"` - NetworkLastModified int64 `json:"networklastmodified" bson:"networklastmodified"` + NodesLastModified int64 `json:"nodeslastmodified" bson:"nodeslastmodified" swaggertype:"primitive,integer" format:"int64"` + NetworkLastModified int64 `json:"networklastmodified" bson:"networklastmodified" swaggertype:"primitive,integer" format:"int64"` DefaultInterface string `json:"defaultinterface" bson:"defaultinterface" validate:"min=1,max=35"` DefaultListenPort int32 `json:"defaultlistenport,omitempty" bson:"defaultlistenport,omitempty" validate:"omitempty,min=1024,max=65535"` NodeLimit int32 `json:"nodelimit" bson:"nodelimit"` diff --git a/models/node.go b/models/node.go index 0ca699dbb..c9e55e5d5 100644 --- a/models/node.go +++ b/models/node.go @@ -11,6 +11,19 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) +type NodeStatus string + +const ( + OnlineSt NodeStatus = "online" + OfflineSt NodeStatus = "offline" + WarningSt NodeStatus = "warning" + ErrorSt NodeStatus = "error" + UnKnown NodeStatus = "unknown" +) + +// LastCheckInThreshold - if node's checkin more than this threshold,then node is declared as offline +const LastCheckInThreshold = time.Minute * 10 + const ( // NODE_SERVER_NAME - the default server name NODE_SERVER_NAME = "netmaker" @@ -103,6 +116,7 @@ type Node struct { IsStatic bool `json:"is_static"` IsUserNode bool `json:"is_user_node"` StaticNode ExtClient `json:"static_node"` + Status NodeStatus `json:"node_status"` } // LegacyNode - legacy struct for node model @@ -123,10 +137,10 @@ type LegacyNode struct { IsHub string `json:"ishub" bson:"ishub" yaml:"ishub" validate:"checkyesorno"` AccessKey string `json:"accesskey" bson:"accesskey" yaml:"accesskey"` Interface string `json:"interface" bson:"interface" yaml:"interface"` - LastModified int64 `json:"lastmodified" bson:"lastmodified" yaml:"lastmodified"` - ExpirationDateTime int64 `json:"expdatetime" bson:"expdatetime" yaml:"expdatetime"` - LastPeerUpdate int64 `json:"lastpeerupdate" bson:"lastpeerupdate" yaml:"lastpeerupdate"` - LastCheckIn int64 `json:"lastcheckin" bson:"lastcheckin" yaml:"lastcheckin"` + LastModified int64 `json:"lastmodified" bson:"lastmodified" yaml:"lastmodified" swaggertype:"primitive,integer" format:"int64"` + ExpirationDateTime int64 `json:"expdatetime" bson:"expdatetime" yaml:"expdatetime" swaggertype:"primitive,integer" format:"int64"` + LastPeerUpdate int64 `json:"lastpeerupdate" bson:"lastpeerupdate" yaml:"lastpeerupdate" swaggertype:"primitive,integer" format:"int64"` + LastCheckIn int64 `json:"lastcheckin" bson:"lastcheckin" yaml:"lastcheckin" swaggertype:"primitive,integer" format:"int64"` MacAddress string `json:"macaddress" bson:"macaddress" yaml:"macaddress"` Password string `json:"password" bson:"password" yaml:"password" validate:"required,min=6"` Network string `json:"network" bson:"network" yaml:"network" validate:"network_exists"` @@ -201,6 +215,19 @@ func (node *Node) PrimaryAddress() string { return node.Address6.IP.String() } +func (node *Node) AddressIPNet4() net.IPNet { + return net.IPNet{ + IP: node.Address.IP, + Mask: net.CIDRMask(32, 32), + } +} +func (node *Node) AddressIPNet6() net.IPNet { + return net.IPNet{ + IP: node.Address6.IP, + Mask: net.CIDRMask(128, 128), + } +} + // ExtClient.PrimaryAddress - returns ipv4 IPNet format func (extPeer *ExtClient) AddressIPNet4() net.IPNet { return net.IPNet{ diff --git a/models/structs.go b/models/structs.go index 2a723adf9..e929e7fc7 100644 --- a/models/structs.go +++ b/models/structs.go @@ -196,7 +196,7 @@ type ServerUpdateData struct { // also contains assymetrical encryption pub/priv keys for any server traffic type Telemetry struct { UUID string `json:"uuid" bson:"uuid"` - LastSend int64 `json:"lastsend" bson:"lastsend"` + LastSend int64 `json:"lastsend" bson:"lastsend" swaggertype:"primitive,integer" format:"int64"` TrafficKeyPriv []byte `json:"traffickeypriv" bson:"traffickeypriv"` TrafficKeyPub []byte `json:"traffickeypub" bson:"traffickeypub"` } @@ -267,6 +267,8 @@ type ServerConfig struct { TrafficKey []byte `yaml:"traffickey"` MetricInterval string `yaml:"metric_interval"` ManageDNS bool `yaml:"manage_dns"` + Stun bool `yaml:"stun"` + StunServers string `yaml:"stun_servers"` DefaultDomain string `yaml:"default_domain"` } diff --git a/mq/emqx_on_prem.go b/mq/emqx_on_prem.go index b9cd690cc..ff8c9bf1f 100644 --- a/mq/emqx_on_prem.go +++ b/mq/emqx_on_prem.go @@ -261,7 +261,7 @@ func (e *EmqxOnPrem) CreateDefaultAllowRule() error { if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, servercfg.GetEmqxRestEndpoint()+"/api/v5/authorization/sources/built_in_database/all", bytes.NewReader(payload)) + req, err := http.NewRequest(http.MethodPost, servercfg.GetEmqxRestEndpoint()+"/api/v5/authorization/sources/built_in_database/rules/all", bytes.NewReader(payload)) if err != nil { return err } diff --git a/mq/migrate.go b/mq/migrate.go index ec8b4afb0..7919e829e 100644 --- a/mq/migrate.go +++ b/mq/migrate.go @@ -88,10 +88,32 @@ func SendPullSYN() error { Host: host, } msg, _ := json.Marshal(hostUpdate) - encrypted, encryptErr := encryptMsg(&host, msg) - if encryptErr != nil { + var encrypted []byte + var encryptErr error + vlt, err := logic.VersionLessThan(host.Version, "v0.30.0") + if err != nil { + slog.Warn("error checking version less than", "warn", err) continue } + if vlt { + encrypted, encryptErr = encryptMsg(&host, msg) + if encryptErr != nil { + slog.Warn("error encrypt with encryptMsg", "warn", encryptErr) + continue + } + } else { + zipped, err := compressPayload(msg) + if err != nil { + slog.Warn("error compressing message", "warn", err) + continue + } + encrypted, encryptErr = encryptAESGCM(host.TrafficKeyPublic[0:32], zipped) + if encryptErr != nil { + slog.Warn("error encrypt with encryptMsg", "warn", encryptErr) + continue + } + } + logger.Log(0, "sending pull syn to", host.Name) mqclient.Publish(fmt.Sprintf("host/update/%s/%s", hostUpdate.Host.ID.String(), servercfg.GetServer()), 0, true, encrypted) } diff --git a/mq/mq.go b/mq/mq.go index bb3a7d538..948a8d224 100644 --- a/mq/mq.go +++ b/mq/mq.go @@ -35,7 +35,7 @@ func setMqOptions(user, password string, opts *mqtt.ClientOptions) { opts.SetConnectRetry(true) opts.SetCleanSession(true) opts.SetConnectRetryInterval(time.Second * 1) - opts.SetKeepAlive(time.Second * 10) + opts.SetKeepAlive(time.Second * 15) opts.SetOrderMatters(false) opts.SetWriteTimeout(time.Minute) } diff --git a/mq/publishers.go b/mq/publishers.go index 3b47390a8..28ef1935d 100644 --- a/mq/publishers.go +++ b/mq/publishers.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" @@ -14,11 +15,9 @@ import ( "golang.org/x/exp/slog" ) -var batchSize = servercfg.GetPeerUpdateBatchSize() -var batchUpdate = servercfg.GetBatchPeerUpdate() - // PublishPeerUpdate --- determines and publishes a peer update to all the hosts func PublishPeerUpdate(replacePeers bool) error { + if !servercfg.IsMessageQueueBackend() { return nil } @@ -37,35 +36,20 @@ func PublishPeerUpdate(replacePeers bool) error { return err } - //if batch peer update disabled - if !batchUpdate { - for _, host := range hosts { - host := host - go func(host models.Host) { - if err = PublishSingleHostPeerUpdate(&host, allNodes, nil, nil, replacePeers, nil); err != nil { - logger.Log(1, "failed to publish peer update to host", host.ID.String(), ": ", err.Error()) + for _, host := range hosts { + host := host + time.Sleep(5 * time.Millisecond) + go func(host models.Host) { + if err = PublishSingleHostPeerUpdate(&host, allNodes, nil, nil, replacePeers, nil); err != nil { + id := host.Name + if host.ID != uuid.Nil { + id = host.ID.String() } - }(host) - } - return nil + slog.Error("failed to publish peer update to host", id, ": ", err) + } + }(host) } - //if batch peer update enabled - batchHost := BatchItems(hosts, batchSize) - var wg sync.WaitGroup - for _, v := range batchHost { - hostLen := len(v) - wg.Add(hostLen) - for i := 0; i < hostLen; i++ { - host := hosts[i] - go func(host models.Host) { - if err = PublishSingleHostPeerUpdate(&host, allNodes, nil, nil, replacePeers, &wg); err != nil { - logger.Log(1, "failed to publish peer update to host", host.ID.String(), ": ", err.Error()) - } - }(host) - } - wg.Wait() - } return nil } diff --git a/mq/util.go b/mq/util.go index a38cd7d75..e9871ccfb 100644 --- a/mq/util.go +++ b/mq/util.go @@ -1,8 +1,14 @@ package mq import ( + "bytes" + "compress/gzip" + "crypto/aes" + "crypto/cipher" + "crypto/rand" "errors" "fmt" + "io" "math" "strings" "time" @@ -66,6 +72,39 @@ func BatchItems[T any](items []T, batchSize int) [][]T { return batches } +func compressPayload(data []byte) ([]byte, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(data); err != nil { + return nil, err + } + zw.Close() + return buf.Bytes(), nil +} +func encryptAESGCM(key, plaintext []byte) ([]byte, error) { + // Create AES block cipher + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + // Create GCM (Galois/Counter Mode) cipher + aesGCM, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + // Create a random nonce + nonce := make([]byte, aesGCM.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + // Encrypt the data + ciphertext := aesGCM.Seal(nonce, nonce, plaintext, nil) + return ciphertext, nil +} + func encryptMsg(host *models.Host, msg []byte) ([]byte, error) { if host.OS == models.OS_Types.IoT { return msg, nil @@ -96,10 +135,29 @@ func encryptMsg(host *models.Host, msg []byte) ([]byte, error) { func publish(host *models.Host, dest string, msg []byte) error { - encrypted, encryptErr := encryptMsg(host, msg) - if encryptErr != nil { - return encryptErr + var encrypted []byte + var encryptErr error + vlt, err := logic.VersionLessThan(host.Version, "v0.30.0") + if err != nil { + slog.Warn("error checking version less than", "error", err) + return err } + if vlt { + encrypted, encryptErr = encryptMsg(host, msg) + if encryptErr != nil { + return encryptErr + } + } else { + zipped, err := compressPayload(msg) + if err != nil { + return err + } + encrypted, encryptErr = encryptAESGCM(host.TrafficKeyPublic[0:32], zipped) + if encryptErr != nil { + return encryptErr + } + } + if mqclient == nil || !mqclient.IsConnectionOpen() { return errors.New("cannot publish ... mqclient not connected") } diff --git a/pro/controllers/users.go b/pro/controllers/users.go index 86787b79b..554b7326c 100644 --- a/pro/controllers/users.go +++ b/pro/controllers/users.go @@ -486,7 +486,7 @@ func updateUserGroup(w http.ResponseWriter, r *http.Request) { // @Summary Delete user group. // @Router /api/v1/user/group [delete] // @Tags Users -// @Param group_id param string true "group id required to delete the role" +// @Param group_id query string true "group id required to delete the role" // @Success 200 {string} string // @Failure 500 {object} models.ErrorResponse func deleteUserGroup(w http.ResponseWriter, r *http.Request) { @@ -517,7 +517,7 @@ func deleteUserGroup(w http.ResponseWriter, r *http.Request) { // @Summary lists all user roles. // @Router /api/v1/user/roles [get] // @Tags Users -// @Param role_id param string true "roleid required to get the role details" +// @Param role_id query string true "roleid required to get the role details" // @Success 200 {object} []models.UserRolePermissionTemplate // @Failure 500 {object} models.ErrorResponse func ListRoles(w http.ResponseWriter, r *http.Request) { @@ -543,7 +543,7 @@ func ListRoles(w http.ResponseWriter, r *http.Request) { // @Summary Get user role permission template. // @Router /api/v1/user/role [get] // @Tags Users -// @Param role_id param string true "roleid required to get the role details" +// @Param role_id query string true "roleid required to get the role details" // @Success 200 {object} models.UserRolePermissionTemplate // @Failure 500 {object} models.ErrorResponse func getRole(w http.ResponseWriter, r *http.Request) { @@ -566,7 +566,7 @@ func getRole(w http.ResponseWriter, r *http.Request) { // @Summary Create user role permission template. // @Router /api/v1/user/role [post] // @Tags Users -// @Param body models.UserRolePermissionTemplate true "user role template" +// @Param body body models.UserRolePermissionTemplate true "user role template" // @Success 200 {object} models.UserRolePermissionTemplate // @Failure 500 {object} models.ErrorResponse func createRole(w http.ResponseWriter, r *http.Request) { @@ -596,8 +596,8 @@ func createRole(w http.ResponseWriter, r *http.Request) { // @Summary Update user role permission template. // @Router /api/v1/user/role [put] // @Tags Users -// @Param body models.UserRolePermissionTemplate true "user role template" -// @Success 200 {object} userBodyResponse +// @Param body body models.UserRolePermissionTemplate true "user role template" +// @Success 200 {object} models.UserRolePermissionTemplate // @Failure 500 {object} models.ErrorResponse func updateRole(w http.ResponseWriter, r *http.Request) { var userRole models.UserRolePermissionTemplate @@ -632,7 +632,7 @@ func updateRole(w http.ResponseWriter, r *http.Request) { // @Summary Delete user role permission template. // @Router /api/v1/user/role [delete] // @Tags Users -// @Param role_id param string true "roleid required to delete the role" +// @Param role_id query string true "roleid required to delete the role" // @Success 200 {string} string // @Failure 500 {object} models.ErrorResponse func deleteRole(w http.ResponseWriter, r *http.Request) { diff --git a/pro/initialize.go b/pro/initialize.go index f6ae65670..55b9c868c 100644 --- a/pro/initialize.go +++ b/pro/initialize.go @@ -140,6 +140,7 @@ func InitPro() { logic.IntialiseGroups = proLogic.UserGroupsInit logic.AddGlobalNetRolesToAdmins = proLogic.AddGlobalNetRolesToAdmins logic.GetUserGroupsInNetwork = proLogic.GetUserGroupsInNetwork + logic.GetNodeStatus = proLogic.GetNodeStatus } func retrieveProLogo() string { diff --git a/pro/license.go b/pro/license.go index 50825d8a3..c623a2409 100644 --- a/pro/license.go +++ b/pro/license.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/gravitl/netmaker/utils" "io" "net/http" "time" @@ -205,55 +206,81 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, bool return nil, false, err } - req, err := http.NewRequest( - http.MethodPost, - proLogic.GetAccountsHost()+"/api/v1/license/validate", - bytes.NewReader(requestBody), - ) - if err != nil { - return nil, false, err - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Accept", "application/json") - client := &http.Client{} - validateResponse, err := client.Do(req) - if err != nil { // check cache - slog.Warn("proceeding with cached response, Netmaker API may be down") - cachedResp, err := getCachedResponse() - return cachedResp, false, err - } - defer validateResponse.Body.Close() - code := validateResponse.StatusCode - - // if we received a 200, cache the response locally - if code == http.StatusOK { - body, err := io.ReadAll(validateResponse.Body) - if err != nil { - slog.Warn("failed to parse response", "error", err) - return nil, false, err - } - if err := cacheResponse(body); err != nil { - slog.Warn("failed to cache response", "error", err) - } - return body, false, nil + var validateResponse *http.Response + var validationResponse []byte + var timedOut bool + + validationRetries := utils.RetryStrategy{ + WaitTime: time.Second * 5, + WaitTimeIncrease: time.Second * 2, + MaxTries: 15, + Wait: func(duration time.Duration) { + time.Sleep(duration) + }, + Try: func() error { + req, err := http.NewRequest( + http.MethodPost, + proLogic.GetAccountsHost()+"/api/v1/license/validate", + bytes.NewReader(requestBody), + ) + if err != nil { + return err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + client := &http.Client{} + + validateResponse, err = client.Do(req) + if err != nil { + slog.Warn(fmt.Sprintf("error while validating license key: %v", err)) + return err + } + + if validateResponse.StatusCode == http.StatusServiceUnavailable || + validateResponse.StatusCode == http.StatusGatewayTimeout || + validateResponse.StatusCode == http.StatusBadGateway { + timedOut = true + return errors.New("failed to reach netmaker api") + } + + return nil + }, + OnMaxTries: func() { + slog.Warn("proceeding with cached response, Netmaker API may be down") + validationResponse, err = getCachedResponse() + timedOut = false + }, + OnSuccess: func() { + defer validateResponse.Body.Close() + + // if we received a 200, cache the response locally + if validateResponse.StatusCode == http.StatusOK { + validationResponse, err = io.ReadAll(validateResponse.Body) + if err != nil { + slog.Warn("failed to parse response", "error", err) + validationResponse = nil + timedOut = false + return + } + + if err := cacheResponse(validationResponse); err != nil { + slog.Warn("failed to cache response", "error", err) + } + } else { + // at this point the backend returned some undesired state + + // inform failure via logs + body, _ := io.ReadAll(validateResponse.Body) + err = fmt.Errorf("could not validate license with validation backend (status={%d}, body={%s})", + validateResponse.StatusCode, string(body)) + slog.Warn(err.Error()) + } + }, } - // at this point the backend returned some undesired state - - // inform failure via logs - body, _ := io.ReadAll(validateResponse.Body) - err = fmt.Errorf("could not validate license with validation backend (status={%d}, body={%s})", - validateResponse.StatusCode, string(body)) - slog.Warn(err.Error()) - - // try to use cache if we had a temporary error - if code == http.StatusServiceUnavailable || code == http.StatusGatewayTimeout || code == http.StatusBadGateway { - slog.Warn("Netmaker API may be down, will retry later...", "code", code) - return nil, true, nil - } + validationRetries.DoStrategy() - // at this point the error is irreversible, return it - return nil, false, err + return validationResponse, timedOut, err } func cacheResponse(response []byte) error { diff --git a/pro/logic/status.go b/pro/logic/status.go new file mode 100644 index 000000000..39dccd269 --- /dev/null +++ b/pro/logic/status.go @@ -0,0 +1,199 @@ +package logic + +import ( + "time" + + "github.com/gravitl/netmaker/logic" + "github.com/gravitl/netmaker/models" +) + +func getNodeStatusOld(node *models.Node) { + // On CE check only last check-in time + if node.IsStatic { + if !node.StaticNode.Enabled { + node.Status = models.OfflineSt + return + } + node.Status = models.OnlineSt + return + } + if time.Since(node.LastCheckIn) > time.Minute*10 { + node.Status = models.OfflineSt + return + } + node.Status = models.OnlineSt +} + +func GetNodeStatus(node *models.Node, defaultEnabledPolicy bool) { + + if time.Since(node.LastCheckIn) > models.LastCheckInThreshold { + node.Status = models.OfflineSt + return + } + if node.IsStatic { + if !node.StaticNode.Enabled { + node.Status = models.OfflineSt + return + } + // check extclient connection from metrics + ingressMetrics, err := GetMetrics(node.StaticNode.IngressGatewayID) + if err != nil || ingressMetrics == nil || ingressMetrics.Connectivity == nil { + node.Status = models.UnKnown + return + } + if metric, ok := ingressMetrics.Connectivity[node.StaticNode.ClientID]; ok { + if metric.Connected { + node.Status = models.OnlineSt + return + } else { + node.Status = models.OfflineSt + return + } + } + node.Status = models.UnKnown + return + } + host, err := logic.GetHost(node.HostID.String()) + if err != nil { + node.Status = models.UnKnown + return + } + vlt, err := logic.VersionLessThan(host.Version, "v0.30.0") + if err != nil { + node.Status = models.UnKnown + return + } + if vlt { + getNodeStatusOld(node) + return + } + metrics, err := logic.GetMetrics(node.ID.String()) + if err != nil { + return + } + if metrics == nil || metrics.Connectivity == nil { + if time.Since(node.LastCheckIn) < models.LastCheckInThreshold { + node.Status = models.OnlineSt + return + } + } + // if node.IsFailOver { + // if time.Since(node.LastCheckIn) < models.LastCheckInThreshold { + // node.Status = models.OnlineSt + // return + // } + // } + // If all Peers are able to reach me and and the peer is not able to reached by any peer then return online + /* 1. FailOver Exists + a. check connectivity to failover Node - if no connection return warning + b. if getting failedover and still no connection to any of the peers - then show error + c. if getting failedOver and has connections to some peers - show warning + 2. FailOver Doesn't Exist + a. check connectivity to pu + + */ + + // failoverNode, exists := FailOverExists(node.Network) + // if exists && failoverNode.FailedOverBy != uuid.Nil { + // // check connectivity to failover Node + // if metric, ok := metrics.Connectivity[failoverNode.ID.String()]; ok { + // if time.Since(failoverNode.LastCheckIn) < models.LastCheckInThreshold { + // if metric.Connected { + // node.Status = models.OnlineSt + // return + // } else { + // checkPeerConnectivity(node, metrics) + // return + // } + // } + // } else { + // node.Status = models.OnlineSt + // return + // } + + // } + checkPeerConnectivity(node, metrics, defaultEnabledPolicy) + +} + +func checkPeerStatus(node *models.Node, defaultAclPolicy bool) { + peerNotConnectedCnt := 0 + metrics, err := logic.GetMetrics(node.ID.String()) + if err != nil { + return + } + if metrics == nil || metrics.Connectivity == nil { + if time.Since(node.LastCheckIn) < models.LastCheckInThreshold { + node.Status = models.OnlineSt + return + } + } + for peerID, metric := range metrics.Connectivity { + peer, err := logic.GetNodeByID(peerID) + if err != nil { + continue + } + allowed, _ := logic.IsNodeAllowedToCommunicate(*node, peer, false) + if !defaultAclPolicy && !allowed { + continue + } + + if time.Since(peer.LastCheckIn) > models.LastCheckInThreshold { + continue + } + if metric.Connected { + continue + } + if peer.Status == models.ErrorSt { + continue + } + peerNotConnectedCnt++ + + } + if peerNotConnectedCnt == 0 { + node.Status = models.OnlineSt + return + } + if peerNotConnectedCnt == len(metrics.Connectivity) { + node.Status = models.ErrorSt + return + } + node.Status = models.WarningSt +} + +func checkPeerConnectivity(node *models.Node, metrics *models.Metrics, defaultAclPolicy bool) { + peerNotConnectedCnt := 0 + for peerID, metric := range metrics.Connectivity { + peer, err := logic.GetNodeByID(peerID) + if err != nil { + continue + } + allowed, _ := logic.IsNodeAllowedToCommunicate(*node, peer, false) + if !defaultAclPolicy && !allowed { + continue + } + + if time.Since(peer.LastCheckIn) > models.LastCheckInThreshold { + continue + } + if metric.Connected { + continue + } + // check if peer is in error state + checkPeerStatus(&peer, defaultAclPolicy) + if peer.Status == models.ErrorSt { + continue + } + peerNotConnectedCnt++ + + } + if peerNotConnectedCnt == 0 { + node.Status = models.OnlineSt + return + } + if peerNotConnectedCnt == len(metrics.Connectivity) { + node.Status = models.ErrorSt + return + } + node.Status = models.WarningSt +} diff --git a/pro/logic/user_mgmt.go b/pro/logic/user_mgmt.go index 2a21a8bdb..daad932af 100644 --- a/pro/logic/user_mgmt.go +++ b/pro/logic/user_mgmt.go @@ -40,7 +40,7 @@ var NetworkAdminAllPermissionTemplate = models.UserRolePermissionTemplate{ var NetworkUserAllPermissionTemplate = models.UserRolePermissionTemplate{ ID: models.UserRoleID(fmt.Sprintf("global-%s", models.NetworkUser)), Name: "Network Users", - MetaData: "cannot access the admin console, but can connect to nodes in your networks via Remote Access Client.", + MetaData: "Can connect to nodes in your networks via Remote Access Client.", Default: true, FullAccess: false, NetworkID: models.AllNetworks, @@ -131,7 +131,7 @@ func UserGroupsInit() { models.UserRoleID(fmt.Sprintf("global-%s", models.NetworkUser)): {}, }, }, - MetaData: "cannot access the admin console, but can connect to nodes in your networks via Remote Access Client.", + MetaData: "Provides read-only dashboard access to platform users and allows connection to network nodes via the Remote Access Client.", } d, _ := json.Marshal(NetworkGlobalAdminGroup) database.Insert(NetworkGlobalAdminGroup.ID.String(), string(d), database.USER_GROUPS_TABLE_NAME) @@ -156,7 +156,7 @@ func CreateDefaultNetworkRolesAndGroups(netID models.NetworkID) { var NetworkUserPermissionTemplate = models.UserRolePermissionTemplate{ ID: models.UserRoleID(fmt.Sprintf("%s-%s", netID, models.NetworkUser)), Name: fmt.Sprintf("%s User", netID), - MetaData: fmt.Sprintf("cannot access the admin console, but can connect to nodes in your network `%s` via Remote Access Client.", netID), + MetaData: fmt.Sprintf("Can connect to nodes in your network `%s` via Remote Access Client.", netID), Default: true, FullAccess: false, NetworkID: netID, @@ -233,7 +233,7 @@ func CreateDefaultNetworkRolesAndGroups(netID models.NetworkID) { models.UserRoleID(fmt.Sprintf("%s-%s", netID, models.NetworkUser)): {}, }, }, - MetaData: fmt.Sprintf("cannot access the admin console, but can connect to nodes in your network `%s` via Remote Access Client.", netID), + MetaData: fmt.Sprintf("Can connect to nodes in your network `%s` via Remote Access Client. Platform users will have read-only access to the the dashboard.", netID), } d, _ = json.Marshal(NetworkAdminGroup) database.Insert(NetworkAdminGroup.ID.String(), string(d), database.USER_GROUPS_TABLE_NAME) diff --git a/pro/types.go b/pro/types.go index 213d77bf8..4c1c85bbe 100644 --- a/pro/types.go +++ b/pro/types.go @@ -18,7 +18,7 @@ var errValidation = errors.New(license_validation_err_msg) // LicenseKey - the license key struct representation with associated data type LicenseKey struct { LicenseValue string `json:"license_value"` // actual (public) key and the unique value for the key - Expiration int64 `json:"expiration"` + Expiration int64 `json:"expiration" swaggertype:"primitive,integer" format:"int64"` UsageServers int `json:"limit_servers"` UsageUsers int `json:"limit_users"` UsageClients int `json:"limit_clients"` diff --git a/release.md b/release.md index f2a6d82aa..80f44c1d3 100644 --- a/release.md +++ b/release.md @@ -1,21 +1,21 @@ -# Netmaker v0.26.0 +# Netmaker v0.30.0 ## Whats New ✨ -- New ACLs and Tag Management System -- Managed DNS system (Linux) -- Simplified User Mgmt With Default Roles and Groups (Hidden away network roles) -- New Add a Node Flow for netclient and static wireguard files +- Advanced ACL Rules - port, protocol and traffic direction +- Reduced Firewall Requirements To One Single Port (443 udp/tcp) +- Option to Turn off STUN or specify custom stun servers +- Improved Connectivity Status Indicator with real-time troubleshooting help. ## What's Fixed/Improved 🛠 - Metrics Data +- Optimised MQ message size - FailOver Stability Fixes - Scalability Fixes +- Duplicate Node IP check on update ## Known Issues 🐞 -- Adding Custom Private/Public Key For Remote Access Gw Clients Doesn't Get Propagated To Other Peers. - IPv6 DNS Entries Are Not Working. - Stale Peer On The Interface, When Forced Removed From Multiple Networks At Once. -- Can Still Ping The Domain Name Even When The DNS Toggle Is Switched Off. - WireGuard DNS issue on most flavours of Ubuntu 24.04 and some other newer Linux distributions. The issue is affecting the Remote Access Client (RAC) and the plain WireGuard external clients. Workaround can be found here https://help.netmaker.io/en/articles/9612016-extclient-rac-dns-issue-on-ubuntu-24-04. diff --git a/scripts/netmaker.default.env b/scripts/netmaker.default.env index b8a65a1fa..d534abacc 100644 --- a/scripts/netmaker.default.env +++ b/scripts/netmaker.default.env @@ -86,11 +86,11 @@ EMAIL_SENDER_ADDR= EMAIL_SENDER_USER= # sender smtp password EMAIL_SENDER_PASSWORD= -# if batch peer update enable or not -PEER_UPDATE_BATCH=true -# batch peer update size when PEER_UPDATE_BATCH is enabled -PEER_UPDATE_BATCH_SIZE=50 # default domain for internal DNS lookup DEFAULT_DOMAIN=netmaker.hosted # managed dns setting, set to true to resolve dns entries on netmaker network MANAGE_DNS=false +# set to true, old acl is supported, otherwise, old acl is disabled +OLD_ACL_SUPPORT=true +# if STUN is set to true, hole punch is called +STUN=true diff --git a/scripts/nm-quick.sh b/scripts/nm-quick.sh index 232eb3f02..dcd6398ef 100755 --- a/scripts/nm-quick.sh +++ b/scripts/nm-quick.sh @@ -127,7 +127,7 @@ setup_netclient() { ./netclient install echo "Register token: $TOKEN" sleep 2 - netclient join -t $TOKEN + netclient join -t $TOKEN --static-port -p 443 echo "waiting for netclient to become available" local found=false @@ -251,10 +251,8 @@ save_config() { ( if [ "$INSTALL_TYPE" = "pro" ]; then save_config_item NETMAKER_TENANT_ID "$NETMAKER_TENANT_ID" save_config_item LICENSE_KEY "$LICENSE_KEY" - if [ "$UPGRADE_FLAG" = "yes" ];then - save_config_item METRICS_EXPORTER "on" - save_config_item PROMETHEUS "on" - fi + save_config_item METRICS_EXPORTER "on" + save_config_item PROMETHEUS "on" save_config_item SERVER_IMAGE_TAG "$IMAGE_TAG-ee" else save_config_item METRICS_EXPORTER "off" @@ -559,7 +557,7 @@ set_install_vars() { echo " api.$NETMAKER_BASE_DOMAIN" echo " broker.$NETMAKER_BASE_DOMAIN" - if [ "$UPGRADE_FLAG" = "yes" ]; then + if [ "$INSTALL_TYPE" = "pro" ]; then echo " prometheus.$NETMAKER_BASE_DOMAIN" echo " netmaker-exporter.$NETMAKER_BASE_DOMAIN" echo " grafana.$NETMAKER_BASE_DOMAIN" @@ -632,13 +630,12 @@ install_netmaker() { if [ "$INSTALL_TYPE" = "pro" ]; then local COMPOSE_OVERRIDE_URL="$BASE_URL/compose/docker-compose.pro.yml" local CADDY_URL="$BASE_URL/docker/Caddyfile-pro" - fi - wget -qO "$SCRIPT_DIR"/docker-compose.yml $COMPOSE_URL - if [ "$UPGRADE_FLAG" = "yes" ]; then wget -qO "$SCRIPT_DIR"/docker-compose.override.yml $COMPOSE_OVERRIDE_URL elif [ -a "$SCRIPT_DIR"/docker-compose.override.yml ]; then rm -f "$SCRIPT_DIR"/docker-compose.override.yml fi + wget -qO "$SCRIPT_DIR"/docker-compose.yml $COMPOSE_URL + wget -qO "$SCRIPT_DIR"/Caddyfile "$CADDY_URL" wget -qO "$SCRIPT_DIR"/netmaker.default.env "$BASE_URL/scripts/netmaker.default.env" wget -qO "$SCRIPT_DIR"/mosquitto.conf "$BASE_URL/docker/mosquitto.conf" diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 2cd80e983..c928adaa9 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -76,6 +76,7 @@ func GetServerConfig() config.ServerConfig { cfg.Database = GetDB() cfg.Platform = GetPlatform() cfg.Version = GetVersion() + cfg.PublicIp = GetServerHostIP() // == auth config == var authInfo = GetAuthProviderInfo() @@ -94,6 +95,8 @@ func GetServerConfig() config.ServerConfig { cfg.RacAutoDisable = GetRacAutoDisable() cfg.MetricInterval = GetMetricInterval() cfg.ManageDNS = GetManageDNS() + cfg.Stun = IsStunEnabled() + cfg.StunServers = GetStunServers() cfg.DefaultDomain = GetDefaultDomain() return cfg } @@ -140,6 +143,8 @@ func GetServerInfo() models.ServerConfig { cfg.IsPro = IsPro cfg.MetricInterval = GetMetricInterval() cfg.ManageDNS = GetManageDNS() + cfg.Stun = IsStunEnabled() + cfg.StunServers = GetStunServers() cfg.DefaultDomain = GetDefaultDomain() return cfg } @@ -176,6 +181,11 @@ func GetVersion() string { return Version } +// GetServerHostIP - fetches server IP +func GetServerHostIP() string { + return os.Getenv("SERVER_HOST") +} + // GetDB - gets the database type func GetDB() string { database := "sqlite" @@ -664,6 +674,14 @@ func GetManageDNS() bool { return enabled } +func IsOldAclEnabled() bool { + enabled := true + if os.Getenv("OLD_ACL_SUPPORT") != "" { + enabled = os.Getenv("OLD_ACL_SUPPORT") == "true" + } + return enabled +} + // GetDefaultDomain - get the default domain func GetDefaultDomain() string { //default netmaker.hosted @@ -686,28 +704,6 @@ func validateDomain(domain string) bool { return exp.MatchString(domain) } -// GetBatchPeerUpdate - if batch peer update -func GetBatchPeerUpdate() bool { - enabled := true - if os.Getenv("PEER_UPDATE_BATCH") != "" { - enabled = os.Getenv("PEER_UPDATE_BATCH") == "true" - } - return enabled -} - -// GetPeerUpdateBatchSize - get the batch size for peer update -func GetPeerUpdateBatchSize() int { - //default 50 - batchSize := 50 - if os.Getenv("PEER_UPDATE_BATCH_SIZE") != "" { - b, e := strconv.Atoi(os.Getenv("PEER_UPDATE_BATCH_SIZE")) - if e == nil && b > 0 && b < 1000 { - batchSize = b - } - } - return batchSize -} - // GetEmqxRestEndpoint - returns the REST API Endpoint of EMQX func GetEmqxRestEndpoint() string { return os.Getenv("EMQX_REST_ENDPOINT") @@ -805,6 +801,19 @@ func IsEndpointDetectionEnabled() bool { return enabled } +// IsStunEnabled - returns true if STUN set to on +func IsStunEnabled() bool { + var enabled = true + if os.Getenv("STUN") != "" { + enabled = os.Getenv("STUN") == "true" + } + return enabled +} + +func GetStunServers() string { + return os.Getenv("STUN_SERVERS") +} + // GetEnvironment returns the environment the server is running in (e.g. dev, staging, prod...) func GetEnvironment() string { if env := os.Getenv("ENVIRONMENT"); env != "" { diff --git a/swagger.yaml b/swagger.yaml index e3d8a276c..a9eb97e14 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1,6 +1,7 @@ definitions: acls.ACL: additionalProperties: + format: int32 type: integer type: object acls.ACLContainer: @@ -41,6 +42,8 @@ definitions: type: string database: type: string + defaultDomain: + type: string deployedByOperator: type: boolean disableRemoteIPCheck: @@ -53,6 +56,12 @@ definitions: type: string egressesLimit: type: integer + email_sender_addr: + type: string + email_sender_password: + type: string + email_sender_user: + type: string emqxRestEndpoint: type: string endpoint_detection: @@ -66,11 +75,14 @@ definitions: ingressesLimit: type: integer jwtValidityDuration: - $ref: '#/definitions/time.Duration' + format: int64 + type: integer licenseValue: type: string machinesLimit: type: integer + manageDNS: + type: boolean masterKey: type: string messageQueueBackend: @@ -99,6 +111,8 @@ definitions: type: string publicIPService: type: string + publicIp: + type: string racAutoDisable: type: boolean restBackend: @@ -107,12 +121,20 @@ definitions: type: string serverBrokerEndpoint: type: string + smtp_host: + type: string + smtp_port: + type: integer sqlconn: type: string + stun: + type: boolean stunList: type: string stunPort: type: integer + stunServers: + type: string telemetry: type: string turnApiServer: @@ -137,7 +159,12 @@ definitions: models.APIEnrollmentKey: properties: expiration: + format: int64 type: integer + groups: + items: + type: string + type: array networks: items: type: string @@ -157,6 +184,41 @@ definitions: required: - tags type: object + models.AclRule: + properties: + allowed: + type: boolean + allowed_ports: + items: + type: string + type: array + allowed_protocols: + allOf: + - $ref: '#/definitions/models.Protocol' + description: tcp, udp, etc. + direction: + allOf: + - $ref: '#/definitions/models.AllowedTrafficDirection' + description: single or two-way + id: + type: string + ip_list: + items: + $ref: '#/definitions/net.IPNet' + type: array + ip6_list: + items: + $ref: '#/definitions/net.IPNet' + type: array + type: object + models.AllowedTrafficDirection: + enum: + - 0 + - 1 + type: integer + x-enum-varnames: + - TrafficDirectionUni + - TrafficDirectionBi models.ApiHost: properties: autoupdate: @@ -245,6 +307,7 @@ definitions: type: string type: array expdatetime: + format: int64 type: integer fail_over_peers: additionalProperties: @@ -262,10 +325,18 @@ definitions: $ref: '#/definitions/models.InetNodeReq' ingressdns: type: string + ingressmtu: + type: integer + ingresspersistentkeepalive: + type: integer internetgw_node_id: type: string is_fail_over: type: boolean + is_static: + type: boolean + is_user_node: + type: boolean isegressgateway: type: boolean isingressgateway: @@ -277,10 +348,13 @@ definitions: isrelayed: type: boolean lastcheckin: + format: int64 type: integer lastmodified: + format: int64 type: integer lastpeerupdate: + format: int64 type: integer localaddress: type: string @@ -302,6 +376,14 @@ definitions: type: array server: type: string + static_node: + $ref: '#/definitions/models.ExtClient' + status: + $ref: '#/definitions/models.NodeStatus' + tags: + additionalProperties: + type: object + type: object required: - hostid - id @@ -375,8 +457,14 @@ definitions: type: object models.EnrollmentKey: properties: + default: + type: boolean expiration: type: string + groups: + items: + type: string + type: array networks: items: type: string @@ -418,10 +506,14 @@ definitions: type: array clientid: type: string + country: + type: string deniednodeacls: additionalProperties: type: object type: object + device_name: + type: string dns: type: string enabled: @@ -435,9 +527,12 @@ definitions: ingressgatewayid: type: string lastmodified: + format: int64 type: integer network: type: string + os: + type: string ownerid: type: string postdown: @@ -446,25 +541,64 @@ definitions: type: string privatekey: type: string + public_endpoint: + type: string publickey: type: string remote_access_client_id: description: unique ID (MAC address) of RAC machine type: string + tags: + additionalProperties: + type: object + type: object type: object models.FailOverMeReq: properties: node_id: type: string type: object + models.FwRule: + properties: + allow: + type: boolean + allowed_ports: + items: + type: string + type: array + allowed_protocols: + allOf: + - $ref: '#/definitions/models.Protocol' + description: tcp, udp, etc. + dst_ip: + $ref: '#/definitions/net.IPNet' + src_ip: + $ref: '#/definitions/net.IPNet' + type: object models.FwUpdate: properties: + acl_rules: + additionalProperties: + $ref: '#/definitions/models.AclRule' + type: object + allow_all: + type: boolean egress_info: additionalProperties: $ref: '#/definitions/models.EgressInfo' type: object + ingress_info: + additionalProperties: + $ref: '#/definitions/models.IngressInfo' + type: object is_egress_gw: type: boolean + is_ingress_gw: + type: boolean + networks: + items: + $ref: '#/definitions/net.IPNet' + type: array type: object models.Host: properties: @@ -527,7 +661,8 @@ definitions: os: type: string persistentkeepalive: - $ref: '#/definitions/time.Duration' + format: int64 + type: integer publickey: items: type: integer @@ -684,6 +819,35 @@ definitions: $ref: '#/definitions/models.ReturnUser' type: array type: object + models.IngressInfo: + properties: + allow_all: + type: boolean + egress_ranges: + items: + $ref: '#/definitions/net.IPNet' + type: array + egress_ranges6: + items: + $ref: '#/definitions/net.IPNet' + type: array + ingress_id: + type: string + network: + $ref: '#/definitions/net.IPNet' + network6: + $ref: '#/definitions/net.IPNet' + rules: + items: + $ref: '#/definitions/models.FwRule' + type: array + static_node_ips: + items: + items: + type: integer + type: array + type: array + type: object models.KeyType: enum: - 0 @@ -699,22 +863,34 @@ definitions: models.Metric: properties: actualuptime: - $ref: '#/definitions/time.Duration' + format: int64 + type: integer connected: type: boolean + lasttotalreceived: + format: int64 + type: integer + lasttotalsent: + format: int64 + type: integer latency: + format: int64 type: integer node_name: type: string percentup: type: number totalreceived: + format: int64 type: integer totalsent: + format: int64 type: integer totaltime: + format: int64 type: integer uptime: + format: int64 type: integer type: object models.Metrics: @@ -766,14 +942,22 @@ definitions: minLength: 1 type: string networklastmodified: + format: int64 type: integer nodelimit: type: integer nodeslastmodified: + format: int64 type: integer required: - netid type: object + models.NetworkID: + enum: + - all_networks + type: string + x-enum-varnames: + - AllNetworks models.Node: properties: action: @@ -821,10 +1005,18 @@ definitions: type: string ingressgatewayrange6: type: string + ingressmtu: + type: integer + ingresspersistentkeepalive: + type: integer internetgw_node_id: type: string is_fail_over: type: boolean + is_static: + type: boolean + is_user_node: + type: boolean isegressgateway: type: boolean isingressgateway: @@ -851,6 +1043,8 @@ definitions: type: integer networkrange6: type: number + node_status: + $ref: '#/definitions/models.NodeStatus' ownerid: type: string pendingdelete: @@ -863,6 +1057,12 @@ definitions: type: array server: type: string + static_node: + $ref: '#/definitions/models.ExtClient' + tags: + additionalProperties: + type: object + type: object type: object models.NodeGet: properties: @@ -883,10 +1083,36 @@ definitions: serverconfig: $ref: '#/definitions/models.ServerConfig' type: object + models.NodeStatus: + enum: + - online + - offline + - warning + - error + - unknown + type: string + x-enum-varnames: + - OnlineSt + - OfflineSt + - WarningSt + - ErrorSt + - UnKnown models.PeerMap: additionalProperties: $ref: '#/definitions/models.IDandAddr' type: object + models.Protocol: + enum: + - all + - udp + - tcp + - icmp + type: string + x-enum-varnames: + - ALL + - UDP + - TCP + - ICMP models.RegisterResponse: properties: requested_host: @@ -907,19 +1133,49 @@ definitions: type: object models.ReturnUser: properties: + auth_type: + type: string isadmin: type: boolean issuperadmin: type: boolean last_login_time: type: string + network_roles: + additionalProperties: + additionalProperties: + type: object + type: object + type: object + platform_role_id: + $ref: '#/definitions/models.UserRoleID' remote_gw_ids: + additionalProperties: + type: object + description: deprecated + type: object + user_group_ids: additionalProperties: type: object type: object username: type: string type: object + models.RsrcPermissionScope: + properties: + create: + type: boolean + delete: + type: boolean + read: + type: boolean + self_only: + type: boolean + update: + type: boolean + vpn_access: + type: boolean + type: object models.ServerConfig: properties: Is_EE: @@ -934,8 +1190,12 @@ definitions: type: string coreDNSAddr: type: string + defaultDomain: + type: string dnsmode: type: string + manageDNS: + type: boolean metricInterval: type: string mqpassword: @@ -946,6 +1206,10 @@ definitions: type: string server: type: string + stun: + type: boolean + stunServers: + type: string trafficKey: items: type: integer @@ -996,16 +1260,35 @@ definitions: type: object models.User: properties: + auth_type: + type: string + external_identity_provider_id: + type: string isadmin: + description: deprecated type: boolean issuperadmin: + description: deprecated type: boolean last_login_time: type: string + network_roles: + additionalProperties: + additionalProperties: + type: object + type: object + type: object password: minLength: 5 type: string + platform_role_id: + $ref: '#/definitions/models.UserRoleID' remote_gw_ids: + additionalProperties: + type: object + description: deprecated + type: object + user_group_ids: additionalProperties: type: object type: object @@ -1052,6 +1335,51 @@ definitions: remote_access_gw_id: type: string type: object + models.UserRoleID: + enum: + - super-admin + - admin + - service-user + - platform-user + - network-admin + - network-user + type: string + x-enum-varnames: + - SuperAdminRole + - AdminRole + - ServiceUser + - PlatformUser + - NetworkAdmin + - NetworkUser + models.UserRolePermissionTemplate: + properties: + default: + type: boolean + deny_dashboard_access: + type: boolean + full_access: + type: boolean + global_level_access: + additionalProperties: + additionalProperties: + $ref: '#/definitions/models.RsrcPermissionScope' + type: object + type: object + id: + $ref: '#/definitions/models.UserRoleID' + meta_data: + type: string + name: + type: string + network_id: + $ref: '#/definitions/models.NetworkID' + network_level_access: + additionalProperties: + additionalProperties: + $ref: '#/definitions/models.RsrcPermissionScope' + type: object + type: object + type: object net.IPNet: properties: ip: @@ -1062,6 +1390,7 @@ definitions: mask: description: network mask items: + format: int32 type: integer type: array type: object @@ -1079,42 +1408,6 @@ definitions: type: object netip.AddrPort: type: object - time.Duration: - enum: - - -9223372036854775808 - - 9223372036854775807 - - 1 - - 1000 - - 1000000 - - 1000000000 - - 60000000000 - - 3600000000000 - - -9223372036854775808 - - 9223372036854775807 - - 1 - - 1000 - - 1000000 - - 1000000000 - - 60000000000 - - 3600000000000 - type: integer - x-enum-varnames: - - minDuration - - maxDuration - - Nanosecond - - Microsecond - - Millisecond - - Second - - Minute - - Hour - - minDuration - - maxDuration - - Nanosecond - - Microsecond - - Millisecond - - Second - - Minute - - Hour wgtypes.PeerConfig: properties: allowedIPs: @@ -1129,13 +1422,13 @@ definitions: - $ref: '#/definitions/net.UDPAddr' description: Endpoint specifies the endpoint of this peer entry, if not nil. persistentKeepaliveInterval: - allOf: - - $ref: '#/definitions/time.Duration' description: |- PersistentKeepaliveInterval specifies the persistent keepalive interval for this peer, if not nil. A non-nil value of 0 will clear the persistent keepalive interval. + format: int64 + type: integer presharedKey: description: |- PresharedKey specifies a peer's preshared key configuration, if not nil. @@ -1173,7 +1466,7 @@ info: contact: {} description: NetMaker API Docs title: NetMaker - version: 0.24.3 + version: 0.30.0 paths: /api/dns: get: @@ -1325,6 +1618,26 @@ paths: summary: Gets custom DNS entries associated with a network tags: - DNS + /api/dns/adm/{network}/sync: + post: + consumes: + - application/json + responses: + "200": + description: DNS Sync completed successfully + schema: + type: string + "400": + description: Bad Request + schema: + $ref: '#/definitions/models.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Sync DNS entries for a given network + tags: + - DNS /api/dns/adm/pushdns: post: consumes: @@ -2293,7 +2606,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/models.ErrorResponse' - summary: List users attached to an ingress gateway + summary: List users attached to an remote access gateway tags: - PRO /api/nodes/adm/{network}: @@ -2344,22 +2657,6 @@ paths: summary: Get the server status tags: - Server - /api/users: - get: - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/models.User' - type: array - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/models.ErrorResponse' - summary: Get all users - tags: - - Users /api/users/{username}: delete: parameters: @@ -2467,42 +2764,28 @@ paths: - Users /api/users/{username}/remote_access_gw: get: - consumes: - - application/json parameters: - - description: Username + - description: Username to fetch all the gateways with access in: path name: username required: true type: string - - description: Remote Access Client ID - in: query - name: remote_access_clientid - type: string - - description: Request from mobile - in: query - name: from_mobile - type: boolean - produces: - - application/json responses: "200": description: OK schema: - items: - $ref: '#/definitions/models.UserRemoteGws' - type: array - "400": - description: Bad Request - schema: - $ref: '#/definitions/models.ErrorResponse' + additionalProperties: + items: + $ref: '#/definitions/models.UserRemoteGws' + type: array + type: object "500": description: Internal Server Error schema: $ref: '#/definitions/models.ErrorResponse' - summary: Get user's remote access gateways + summary: Get Users Remote Access Gw. tags: - - PRO + - Users /api/users/{username}/remote_access_gw/{remote_access_gateway_id}: delete: consumes: @@ -2730,6 +3013,93 @@ paths: summary: Approve a pending user tags: - Users + /api/v1/acls: + delete: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Delete Acl + tags: + - ACL + get: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: List Acls in a network + tags: + - ACL + post: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Create Acl + tags: + - ACL + put: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Update Acl + tags: + - ACL + /api/v1/acls/policy_types: + get: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: List Acl Policy types + tags: + - ACL /api/v1/enrollment-keys: get: responses: @@ -2947,6 +3317,24 @@ paths: summary: Delete all legacy nodes from DB. tags: - Nodes + /api/v1/networks/stats: + get: + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/models.SuccessResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + security: + - oauth: [] + summary: Lists all networks with stats + tags: + - Networks /api/v1/node/{network}/failover/reset: post: parameters: @@ -3069,6 +3457,196 @@ paths: summary: Failover me tags: - PRO + /api/v1/tags: + delete: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Delete Tag + tags: + - TAG + get: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: List Tags in a network + tags: + - TAG + post: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Create Tag + tags: + - TAG + put: + consumes: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.SuccessResponse' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Update Tag + tags: + - TAG + /api/v1/user/group: + delete: + parameters: + - description: group id required to delete the role + in: query + name: group_id + required: true + type: string + responses: + "200": + description: OK + schema: + type: string + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Delete user group. + tags: + - Users + /api/v1/user/role: + delete: + parameters: + - description: roleid required to delete the role + in: query + name: role_id + required: true + type: string + responses: + "200": + description: OK + schema: + type: string + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Delete user role permission template. + tags: + - Users + get: + parameters: + - description: roleid required to get the role details + in: query + name: role_id + required: true + type: string + responses: + "200": + description: OK + schema: + $ref: '#/definitions/models.UserRolePermissionTemplate' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Get user role permission template. + tags: + - Users + post: + parameters: + - description: user role template + in: body + name: body + required: true + schema: + $ref: '#/definitions/models.UserRolePermissionTemplate' + responses: + "200": + description: OK + schema: + $ref: '#/definitions/models.UserRolePermissionTemplate' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Create user role permission template. + tags: + - Users + put: + parameters: + - description: user role template + in: body + name: body + required: true + schema: + $ref: '#/definitions/models.UserRolePermissionTemplate' + responses: + "200": + description: OK + schema: + $ref: '#/definitions/models.UserRolePermissionTemplate' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Update user role permission template. + tags: + - Users + /api/v1/user/roles: + get: + parameters: + - description: roleid required to get the role details + in: query + name: role_id + required: true + type: string + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/models.UserRolePermissionTemplate' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: lists all user roles. + tags: + - Users /meshclient/files/{filename}: get: responses: diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 000000000..cb8c41d89 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,41 @@ +package utils + +import "time" + +// RetryStrategy specifies a strategy to retry an operation after waiting a while, +// with hooks for successful and unsuccessful (>=max) tries. +type RetryStrategy struct { + Wait func(time.Duration) + WaitTime time.Duration + WaitTimeIncrease time.Duration + MaxTries int + Try func() error + OnMaxTries func() + OnSuccess func() +} + +// DoStrategy does the retry strategy specified in the struct, waiting before retrying an operator, +// up to a max number of tries, and if executes a success "finalizer" operation if a retry is successful +func (rs RetryStrategy) DoStrategy() { + err := rs.Try() + if err == nil { + rs.OnSuccess() + return + } + + tries := 1 + for { + if tries >= rs.MaxTries { + rs.OnMaxTries() + return + } + rs.Wait(rs.WaitTime) + if err := rs.Try(); err != nil { + tries++ // we tried, increase count + rs.WaitTime += rs.WaitTimeIncrease // for the next time, sleep more + continue // retry + } + rs.OnSuccess() + return + } +}