diff --git a/artifacts/definitions/Server/Utils/ArtifactVerifier.yaml b/artifacts/definitions/Server/Utils/ArtifactVerifier.yaml index b4b412b6eae..4f4ed6deb2a 100644 --- a/artifacts/definitions/Server/Utils/ArtifactVerifier.yaml +++ b/artifacts/definitions/Server/Utils/ArtifactVerifier.yaml @@ -40,19 +40,23 @@ sources: regex='''^name:\s*(.+)''', string=Artifact).g1 LET Files = SELECT OSPath, - read_file(filename=OSPath, length=10000) AS Data + read_file(filename=OSPath) AS Data FROM glob(globs=SearchGlob) + LET Artifacts <= SELECT *, + artifact_set(definition=Data, repository="local") AS Definition + FROM Files + LET Results <= SELECT name, path, PassLogError(Verify=Verify, Path=path) AS passed, Stringify(X=Verify.Errors) AS errors, Stringify(X=Verify.Warnings) AS warnings - FROM foreach(row=Files, + FROM foreach(row=Artifacts, query={ SELECT OSPath AS path, GetName(Artifact=Data) AS name, - verify(artifact=Data) AS Verify + verify(artifact=Data, repository="local") AS Verify FROM scope() }) diff --git a/artifacts/testdata/files/artifacts/caller.yaml b/artifacts/testdata/files/artifacts/caller.yaml new file mode 100644 index 00000000000..ac560106c0d --- /dev/null +++ b/artifacts/testdata/files/artifacts/caller.yaml @@ -0,0 +1,3 @@ +name: Caller +sources: +- query: SELECT * FROM Artifact.Good() diff --git a/artifacts/testdata/files/artifacts/info.yaml b/artifacts/testdata/files/artifacts/info.yaml new file mode 100644 index 00000000000..36a919f4809 --- /dev/null +++ b/artifacts/testdata/files/artifacts/info.yaml @@ -0,0 +1,3 @@ +name: Generic.Client.Info +sources: +- query: SELECT * FROM info() diff --git a/artifacts/testdata/server/testcases/verify.in.yaml b/artifacts/testdata/server/testcases/verify.in.yaml index 256c42dc03b..63b264bdf65 100644 --- a/artifacts/testdata/server/testcases/verify.in.yaml +++ b/artifacts/testdata/server/testcases/verify.in.yaml @@ -18,6 +18,16 @@ Parameters: sources: - query: SELECT * FROM execve(argv=["ls"]) + Override1: | + name: Generic.Client.Info + sources: + - query: SELECT * FROM scope() + + Override2: | + name: Override2 + sources: + - query: SELECT * FROM scope() + Queries: # Basic verification of some artifacts. @@ -35,3 +45,10 @@ Queries: } AS results FROM Artifact.Server.Utils.ArtifactVerifier( SearchGlob=srcDir+"/artifacts/testdata/files/artifacts/*") + + # Check the "disable_override" argument produces an error when overriding a built-in, + # but not when creating a new artifact when True is specified. + - SELECT + verify(artifact=Override1, disable_override=True) AS Override1, + verify(artifact=Override2, disable_override=True) AS Override2 + FROM scope() \ No newline at end of file diff --git a/artifacts/testdata/server/testcases/verify.out.yaml b/artifacts/testdata/server/testcases/verify.out.yaml index 4e5e8dae204..bff4a77f5a9 100644 --- a/artifacts/testdata/server/testcases/verify.out.yaml +++ b/artifacts/testdata/server/testcases/verify.out.yaml @@ -49,18 +49,27 @@ Query: SELECT summary, artifacts, { SELECT *, basename(path=path) as path FROM r Output: [ { "summary": { - "total": 4, - "passed": 2, + "total": 6, + "passed": 4, "failed": 2, "warnings": 1 }, "artifacts": [ + "Caller", "Good", + "Generic.Client.Info", "BrokenYaml", "BrokenQuery", "WarnningExecve" ], "results": [ + { + "name": "Caller", + "passed": true, + "errors": [], + "warnings": [], + "path": "caller.yaml" + }, { "name": "Good", "passed": true, @@ -68,6 +77,13 @@ Output: [ "warnings": [], "path": "good.yaml" }, + { + "name": "Generic.Client.Info", + "passed": true, + "errors": [], + "warnings": [], + "path": "info.yaml" + }, { "name": "BrokenYaml", "passed": false, @@ -99,3 +115,27 @@ Output: [ } ] +# Check the "disable_override" argument produces an error when overriding a built-in, +# but not when creating a new artifact when True is specified. +Query: SELECT verify(artifact=Override1, disable_override=True) AS Override1, verify(artifact=Override2, disable_override=True) AS Override2 FROM scope() +Output: [ + { + "Override1": { + "Artifact": "name: Generic.Client.Info\nsources:\n- query: SELECT * FROM scope()\n", + "Permissions": null, + "Errors": [ + "Unable to override built in artifact Generic.Client.Info" + ], + "Warnings": null, + "Definitions": {} + }, + "Override2": { + "Artifact": "name: Override2\nsources:\n- query: SELECT * FROM scope()\n", + "Permissions": null, + "Errors": null, + "Warnings": null, + "Definitions": {} + } + } +] + diff --git a/bin/verify.go b/bin/verify.go index b424cdb4121..627fa8dd16f 100644 --- a/bin/verify.go +++ b/bin/verify.go @@ -2,22 +2,24 @@ package main import ( "fmt" - "os" + "log" + "path/filepath" + "github.com/Velocidex/ordereddict" errors "github.com/go-errors/errors" - artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" - "www.velocidex.com/golang/velociraptor/constants" logging "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/services/launcher" "www.velocidex.com/golang/velociraptor/startup" - "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/velociraptor/vql/acl_managers" + "www.velocidex.com/golang/vfilter" ) var ( verify = artifact_command.Command("verify", "Verify a set of artifacts") verify_args = verify.Arg("paths", "Paths to artifact yaml files").Required().Strings() verify_allow_override = verify.Flag("builtin", "Allow overriding of built in artifacts").Bool() + verify_issues_only = verify.Flag("issues_only", "If set, we only emit warning and error messages").Bool() ) func doVerify() error { @@ -48,60 +50,89 @@ func doVerify() error { logger := logging.GetLogger(config_obj, &logging.ToolComponent) - // Report all errors and keep going as much as possible. - artifacts := make(map[string]*artifacts_proto.Artifact) - states := make(map[string]*launcher.AnalysisState) - - repository, err := manager.GetGlobalRepository(config_obj) - if err != nil { - return err - } + var artifact_paths []string for _, artifact_path := range *verify_args { - state := launcher.NewAnalysisState(artifact_path) - states[artifact_path] = state - - fd, err := os.Open(artifact_path) + abs, err := filepath.Abs(artifact_path) if err != nil { - state.SetError(err) + logger.Error("verify: could not get absolute path for %v", artifact_path) continue } - data, err := utils.ReadAllWithLimit(fd, constants.MAX_MEMORY) - if err != nil { - state.SetError(err) - continue - } + artifact_paths = append(artifact_paths, abs) + } - a, err := repository.LoadYaml(string(data), services.ArtifactOptions{ - ValidateArtifact: true, - ArtifactIsBuiltIn: *verify_allow_override, - AllowOverridingAlias: true, - }) - if err != nil { - state.SetError(err) - continue - } - artifacts[artifact_path] = a + artifact_logger := &LogWriter{config_obj: sm.Config} + builder := services.ScopeBuilder{ + Config: sm.Config, + ACLManager: acl_managers.NewRoleACLManager(sm.Config, "administrator"), + Logger: log.New(artifact_logger, "", 0), + Env: ordereddict.NewDict(). + Set("Artifacts", artifact_paths). + Set("DisableOverride", !*verify_allow_override), } - for artifact_path, artifact := range artifacts { - state, _ := states[artifact_path] - launcher.VerifyArtifact(ctx, config_obj, - repository, artifact, state) + query := ` + -- Load artifacts into local repository + LET Definitions <= SELECT + Filename, + Data, + artifact_set(definition=Data, repository="local") AS Definition + FROM read_file(filenames=Artifacts) + + -- Verify artifacts from local repository + SELECT Filename, + Result + FROM foreach(row=Definitions, + query={ + SELECT Filename, + verify(artifact=Data, + repository="local", + disable_override=DisableOverride) AS Result + FROM scope() + }) + ` + + scope := manager.BuildScope(builder) + defer scope.Close() + + statements, err := vfilter.MultiParse(query) + if err != nil { + logger.Error("verify: error passing query: %v", query) + return err } var ret error - for artifact_path, state := range states { - if len(state.Errors) == 0 { - logger.Info("Verified %v: OK", artifact_path) - } - for _, err := range state.Errors { - logger.Error("%v: %v", artifact_path, err) - ret = errors.New(err) - } - for _, msg := range state.Warnings { - logger.Info("%v: %v", artifact_path, msg) + for _, vql := range statements { + for row := range vql.Eval(sm.Ctx, scope) { + dict := vfilter.RowToDict(ctx, scope, row) + + artifact_path, pres := dict.GetString("Filename") + if !pres { + continue + } + + result, pres := dict.Get("Result") + if !pres { + continue + } + + state, ok := result.(*launcher.AnalysisState) + if !ok { + continue + } + if len(state.Errors) == 0 { + if !*verify_issues_only { + logger.Info("Verified %v: OK", artifact_path) + } + } + for _, err := range state.Errors { + logger.Error("%v: %v", artifact_path, err) + ret = errors.New(err) + } + for _, msg := range state.Warnings { + logger.Info("%v: %v", artifact_path, msg) + } } } diff --git a/docs/references/vql.yaml b/docs/references/vql.yaml index 224d5053dfe..b30b9b4870c 100644 --- a/docs/references/vql.yaml +++ b/docs/references/vql.yaml @@ -264,6 +264,10 @@ type: string description: Optional tags to attach to the artifact. repeated: true + - name: repository + type: string + description: Add the artifact to this repository, if not set, we add the artifact + to the global repository. category: server metadata: permissions: ARTIFACT_WRITER,SERVER_ARTIFACT_WRITER @@ -11588,6 +11592,14 @@ description: The artifact to verify. This can be an artifact source in yaml or json or the name of an artifact required: true + - name: repository + type: string + description: The repository to use for verification, if not set, we default to + the global repository. + - name: disable_override + type: bool + description: If set, we do not allow override of built-in artifacts (allowed by + default) platforms: - linux_amd64_cgo - windows_amd64_cgo diff --git a/vql/golang/verify.go b/vql/golang/verify.go index 08a299a3988..f72bee1c06b 100644 --- a/vql/golang/verify.go +++ b/vql/golang/verify.go @@ -7,12 +7,15 @@ import ( "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/services/launcher" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + vql_server "www.velocidex.com/golang/velociraptor/vql/server" "www.velocidex.com/golang/vfilter" "www.velocidex.com/golang/vfilter/arg_parser" ) type VerifyFunctionArgs struct { - Artifact string `vfilter:"required,field=artifact,doc=The artifact to verify. This can be an artifact source in yaml or json or the name of an artifact"` + Artifact string `vfilter:"required,field=artifact,doc=The artifact to verify. This can be an artifact source in yaml or json or the name of an artifact"` + Repository string `vfilter:"optional,field=repository,doc=The repository to use for verification, if not set, we default to the global repository."` + DisableOverride bool `vfilter:"optional,field=disable_override,doc=If set, we do not allow override of built-in artifacts (allowed by default)"` } func init() { @@ -55,23 +58,26 @@ This function will verify the artifact and flag any potential errors or warnings state := launcher.NewAnalysisState(arg.Artifact) + if arg.Repository != "" { + cached_any := vql_subsystem.CacheGet(scope, vql_server.REPOSITORY_CACHE_TAG+arg.Repository) + + if cached_repository, ok := cached_any.(services.Repository); ok { + repository = cached_repository + } + } + artifact, pres := repository.Get(ctx, config_obj, arg.Artifact) if !pres { - local_repository := manager.NewRepository() - local_repository.SetParent(repository, config_obj) - - artifact, err = local_repository.LoadYaml(arg.Artifact, + artifact, err = repository.LoadYaml(arg.Artifact, services.ArtifactOptions{ ValidateArtifact: true, - ArtifactIsBuiltIn: true, + ArtifactIsBuiltIn: !arg.DisableOverride, AllowOverridingAlias: true, }) if err != nil { state.SetError(err) return state } - - repository = local_repository } // Verify the artifact diff --git a/vql/server/repository.go b/vql/server/repository.go index 6e38ced4a38..017619f649c 100644 --- a/vql/server/repository.go +++ b/vql/server/repository.go @@ -17,10 +17,15 @@ import ( "www.velocidex.com/golang/vfilter/arg_parser" ) +const ( + REPOSITORY_CACHE_TAG = "__REPOSITORY_" +) + type ArtifactSetFunctionArgs struct { Definition string `vfilter:"optional,field=definition,doc=Artifact definition in YAML"` Prefix string `vfilter:"optional,field=prefix,doc=Optional name prefix (deprecated ignored)"` Tags []string `vfilter:"optional,field=tags,doc=Optional tags to attach to the artifact."` + Repository string `vfilter:"optional,field=repository,doc=Add the artifact to this repository, if not set, we add the artifact to the global repository."` } type ArtifactSetFunction struct{} @@ -87,6 +92,55 @@ func (self *ArtifactSetFunction) Call(ctx context.Context, principal := vql_subsystem.GetPrincipal(scope) + global_repository, err := manager.GetGlobalRepository(config_obj) + if err != nil { + scope.Log("artifact_set: %s", err) + return vfilter.Null{} + } + + if arg.Repository != "" { + var local_repository services.Repository + cached_any := vql_subsystem.CacheGet(scope, REPOSITORY_CACHE_TAG+arg.Repository) + + if cached_repository, ok := cached_any.(services.Repository); ok { + local_repository = cached_repository + } else { + scope.Log("artifact_set: creating new repository '%s'", arg.Repository) + local_repository = manager.NewRepository() + local_repository.SetParent(global_repository, config_obj) + } + + // Determine if this is a built-in artifact + tmp_repository := local_repository.Copy() + built_in := false + + artifact, err := tmp_repository.LoadYaml(arg.Definition, + services.ArtifactOptions{ + ValidateArtifact: true, + ArtifactIsBuiltIn: true, + }) + if err == nil { + if global_artifact, pres := global_repository.Get(ctx, config_obj, artifact.Name); pres { + built_in = global_artifact.BuiltIn + } + } + + definition, err := local_repository.LoadYaml(arg.Definition, + services.ArtifactOptions{ + ValidateArtifact: true, + ArtifactIsBuiltIn: built_in, + }) + if err != nil { + scope.Log("artifact_set: %s", err) + return vfilter.Null{} + } + + scope.Log("artifact_set: added %s to repository '%s'", definition.Name, arg.Repository) + vql_subsystem.CacheSet(scope, REPOSITORY_CACHE_TAG+arg.Repository, local_repository) + + return json.ConvertProtoToOrderedDict(definition) + } + definition, err = manager.SetArtifactFile(ctx, config_obj, principal, arg.Definition, arg.Prefix) if err != nil {