feature: add JSON report to verify command

artifacts/definitions/Server/Utils/ArtifactVerifier.yaml

@@ -40,7 +40,7 @@ sources:
         regex='''^name:\s*(.+)''', string=Artifact).g1
 
     LET Files = SELECT OSPath,
-                       read_file(filename=OSPath, length=10000) AS Data
+                       read_file(filename=OSPath, length=4194304) AS Data
       FROM glob(globs=SearchGlob)
 
     LET Results <= SELECT name,

artifacts/testdata/server/testcases/verify.in.yaml

@@ -18,6 +18,16 @@ Parameters:
     sources:
     - query: SELECT * FROM execve(argv=["ls"])
 
+  Override1: |
+    name: Generic.Client.Info
+    sources:
+    - query: SELECT * FROM scope()
+
+  Override2: |
+    name: Override2
+    sources:
+    - query: SELECT * FROM scope()
+
 
 Queries:
   # Basic verification of some artifacts.
@@ -35,3 +45,10 @@ Queries:
     } AS results
     FROM Artifact.Server.Utils.ArtifactVerifier(
        SearchGlob=srcDir+"/artifacts/testdata/files/artifacts/*")
+
+  # Check the "disable_override" argument produces an error when overriding a built-in,
+  # but not when creating a new artifact when True is specified.
+  - SELECT 
+      verify(artifact=Override1, disable_override=True) AS Override1,
+      verify(artifact=Override2, disable_override=True) AS Override2
+    FROM scope()

artifacts/testdata/server/testcases/verify.out.yaml

@@ -99,3 +99,27 @@ Output: [
  }
 ]
 
+# Check the "disable_override" argument produces an error when overriding a built-in,
+# but not when creating a new artifact when True is specified.
+Query: SELECT verify(artifact=Override1, disable_override=True) AS Override1, verify(artifact=Override2, disable_override=True) AS Override2 FROM scope()
+Output: [
+ {
+  "Override1": {
+   "Artifact": "name: Generic.Client.Info\nsources:\n- query: SELECT * FROM scope()\n",
+   "Permissions": null,
+   "Errors": [
+    "Unable to override built in artifact Generic.Client.Info"
+   ],
+   "Warnings": null,
+   "Definitions": {}
+  },
+  "Override2": {
+   "Artifact": "name: Override2\nsources:\n- query: SELECT * FROM scope()\n",
+   "Permissions": null,
+   "Errors": null,
+   "Warnings": null,
+   "Definitions": {}
+  }
+ }
+]
+

bin/verify.go

@@ -2,16 +2,18 @@ package main
 
 import (
 	"fmt"
-	"os"
+	"log"
+	"path/filepath"
 
+	"github.com/Velocidex/ordereddict"
 	errors "github.com/go-errors/errors"
-	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
-	"www.velocidex.com/golang/velociraptor/constants"
+	"www.velocidex.com/golang/velociraptor/json"
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/services/launcher"
 	"www.velocidex.com/golang/velociraptor/startup"
-	"www.velocidex.com/golang/velociraptor/utils"
+	"www.velocidex.com/golang/velociraptor/vql/acl_managers"
+	"www.velocidex.com/golang/vfilter"
 )
 
 var (
@@ -46,49 +48,77 @@ func doVerify() error {
 		return err
 	}
 
-	logger := logging.GetLogger(config_obj, &logging.ToolComponent)
-
 	// Report all errors and keep going as much as possible.
-	artifacts := make(map[string]*artifacts_proto.Artifact)
 	states := make(map[string]*launcher.AnalysisState)
 
-	repository, err := manager.GetGlobalRepository(config_obj)
-	if err != nil {
-		return err
-	}
+	logger := logging.GetLogger(config_obj, &logging.ToolComponent)
 
-	for _, artifact_path := range *verify_args {
-		state := launcher.NewAnalysisState(artifact_path)
-		states[artifact_path] = state
+	var artifact_paths []string
 
-		fd, err := os.Open(artifact_path)
+	for _, artifact_path := range *verify_args {
+		abs, err := filepath.Abs(artifact_path)
 		if err != nil {
-			state.SetError(err)
+			logger.Error("verify: could not get absolute path for %v", artifact_path)
 			continue
 		}
 
-		data, err := utils.ReadAllWithLimit(fd, constants.MAX_MEMORY)
-		if err != nil {
-			state.SetError(err)
-			continue
-		}
+		artifact_paths = append(artifact_paths, abs)
+	}
 
-		a, err := repository.LoadYaml(string(data), services.ArtifactOptions{
-			ValidateArtifact:     true,
-			ArtifactIsBuiltIn:    *verify_allow_override,
-			AllowOverridingAlias: true,
-		})
-		if err != nil {
-			state.SetError(err)
-			continue
-		}
-		artifacts[artifact_path] = a
+	artifact_logger := &LogWriter{config_obj: sm.Config}
+	builder := services.ScopeBuilder{
+		Config:     sm.Config,
+		ACLManager: acl_managers.NewRoleACLManager(sm.Config, "administrator"),
+		Logger:     log.New(artifact_logger, "", 0),
+		Env: ordereddict.NewDict().
+			Set("Artifacts", artifact_paths).
+			Set("DisableOverride", !*verify_allow_override),
 	}
 
-	for artifact_path, artifact := range artifacts {
-		state, _ := states[artifact_path]
-		launcher.VerifyArtifact(ctx, config_obj,
-			repository, artifact, state)
+	query := `
+		-- Load artifacts into local repository
+		LET Definitions = SELECT Filename, artifact_set(definition=Data, repository="local") AS Definition FROM read_file(filenames=Artifacts)
+
+		-- Verify artifacts from local repository
+		SELECT Filename, Result FROM foreach(
+			row=Definitions,
+			query={ SELECT Filename, verify(artifact=Definition.name, repository="local") AS Result FROM scope() }
+		)
+	`
+
+	scope := manager.BuildScope(builder)
+	defer scope.Close()
+
+	statements, err := vfilter.MultiParse(query)
+	if err != nil {
+		logger.Error("verify: error passing query: %v", query)
+		return err
+	}
+
+	for _, vql := range statements {
+		for row := range vql.Eval(sm.Ctx, scope) {
+			dict := vfilter.RowToDict(ctx, scope, row)
+
+			path, pres := dict.GetString("Filename")
+			if !pres {
+				continue
+			}
+
+			result, pres := dict.Get("Result")
+			if !pres {
+				continue
+			}
+
+			serialized := json.MustMarshalIndent(result)
+			state := &launcher.AnalysisState{}
+			err := json.Unmarshal(serialized, state)
+			if err != nil {
+				logger.Error("verify: could not unmarshal analysis state")
+				continue
+			}
+
+			states[path] = state
+		}
 	}
 
 	var ret error

docs/references/vql.yaml

@@ -264,6 +264,10 @@
     type: string
     description: Optional tags to attach to the artifact.
     repeated: true
+  - name: repository
+    type: string
+    description: Add the artifact to this repository, if not set, we add the artifact
+      to the global repository.
   category: server
   metadata:
     permissions: ARTIFACT_WRITER,SERVER_ARTIFACT_WRITER
@@ -11588,6 +11592,14 @@
     description: The artifact to verify. This can be an artifact source in yaml or
       json or the name of an artifact
     required: true
+  - name: repository
+    type: string
+    description: The repository to use for verification, if not set, we default to
+      the global repository.
+  - name: disable_override
+    type: bool
+    description: If set, we do not allow override of built-in artifacts (allowed by
+      default)
   platforms:
   - linux_amd64_cgo
   - windows_amd64_cgo

vql/golang/verify.go

@@ -12,7 +12,9 @@ import (
 )
 
 type VerifyFunctionArgs struct {
-	Artifact string `vfilter:"required,field=artifact,doc=The artifact to verify. This can be an artifact source in yaml or json or the name of an artifact"`
+	Artifact        string `vfilter:"required,field=artifact,doc=The artifact to verify. This can be an artifact source in yaml or json or the name of an artifact"`
+	Repository      string `vfilter:"optional,field=repository,doc=The repository to use for verification, if not set, we default to the global repository."`
+	DisableOverride bool   `vfilter:"optional,field=disable_override,doc=If set, we do not allow override of built-in artifacts (allowed by default)"`
 }
 
 func init() {
@@ -55,23 +57,26 @@ This function will verify the artifact and flag any potential errors or warnings
 
 				state := launcher.NewAnalysisState(arg.Artifact)
 
+				if arg.Repository != "" {
+					cached_any := vql_subsystem.CacheGet(scope, arg.Repository)
+
+					if cached_repository, ok := cached_any.(services.Repository); ok {
+						repository = cached_repository
+					}
+				}
+
 				artifact, pres := repository.Get(ctx, config_obj, arg.Artifact)
 				if !pres {
-					local_repository := manager.NewRepository()
-					local_repository.SetParent(repository, config_obj)
-
-					artifact, err = local_repository.LoadYaml(arg.Artifact,
+					artifact, err = repository.LoadYaml(arg.Artifact,
 						services.ArtifactOptions{
 							ValidateArtifact:     true,
-							ArtifactIsBuiltIn:    true,
+							ArtifactIsBuiltIn:    !arg.DisableOverride,
 							AllowOverridingAlias: true,
 						})
 					if err != nil {
 						state.SetError(err)
 						return state
 					}
-
-					repository = local_repository
 				}
 
 				// Verify the artifact

vql/server/repository.go

@@ -21,6 +21,7 @@ type ArtifactSetFunctionArgs struct {
 	Definition string   `vfilter:"optional,field=definition,doc=Artifact definition in YAML"`
 	Prefix     string   `vfilter:"optional,field=prefix,doc=Optional name prefix (deprecated ignored)"`
 	Tags       []string `vfilter:"optional,field=tags,doc=Optional tags to attach to the artifact."`
+	Repository string   `vfilter:"optional,field=repository,doc=Add the artifact to this repository, if not set, we add the artifact to the global repository."`
 }
 
 type ArtifactSetFunction struct{}
@@ -87,6 +88,40 @@ func (self *ArtifactSetFunction) Call(ctx context.Context,
 
 	principal := vql_subsystem.GetPrincipal(scope)
 
+	global_repository, err := manager.GetGlobalRepository(config_obj)
+	if err != nil {
+		scope.Log("artifact_set: %s", err)
+		return vfilter.Null{}
+	}
+
+	if arg.Repository != "" {
+		var local_repository services.Repository
+		cached_any := vql_subsystem.CacheGet(scope, arg.Repository)
+
+		if cached_repository, ok := cached_any.(services.Repository); ok {
+			local_repository = cached_repository
+		} else {
+			scope.Log("artifact_set: creating new repository '%s'", arg.Repository)
+			local_repository = manager.NewRepository()
+			local_repository.SetParent(global_repository, config_obj)
+		}
+
+		definition, err := local_repository.LoadYaml(arg.Definition,
+			services.ArtifactOptions{
+				ValidateArtifact:  true,
+				ArtifactIsBuiltIn: true,
+			})
+		if err != nil {
+			scope.Log("artifact_set: %s", err)
+			return vfilter.Null{}
+		}
+
+		scope.Log("artifact_set: added %s to repository '%s'", definition.Name, arg.Repository)
+		vql_subsystem.CacheSet(scope, arg.Repository, local_repository)
+
+		return json.ConvertProtoToOrderedDict(definition)
+	}
+
 	definition, err = manager.SetArtifactFile(ctx,
 		config_obj, principal, arg.Definition, arg.Prefix)
 	if err != nil {