Bugfix: Filestore path types were inferred from the file extension

accessors/api.go

@@ -54,6 +54,11 @@ type OSPath struct {
 	pathspec    *PathSpec
 	serialized  *string
 	Manipulator PathManipulator
+
+	// Opaque data that can be stored in the OSPath. This provides a
+	// mechanism to transport additional data in the OSPath and avoid
+	// having to convert back and forth.
+	Data interface{}
 }
 
 func (self *OSPath) Equal(other *OSPath) bool {

accessors/file_store/accessor.go

@@ -4,7 +4,6 @@ package file_store
 // the generic filestore. This allows us to run globs on the file
 // store regardless of the specific filestore implementation.
 import (
-	"encoding/json"
 	"errors"
 
 	"www.velocidex.com/golang/velociraptor/accessors"
@@ -14,6 +13,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
 	"www.velocidex.com/golang/velociraptor/file_store/path_specs"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/uploads"
 	"www.velocidex.com/golang/velociraptor/utils"
 	"www.velocidex.com/golang/velociraptor/utils/files"
@@ -103,7 +103,8 @@ func (self FileStoreFileSystemAccessor) Lstat(filename string) (
 	return self.LstatWithOSPath(full_path)
 }
 
-func (self FileStoreFileSystemAccessor) LstatWithOSPath(filename *accessors.OSPath) (
+func (self FileStoreFileSystemAccessor) LstatWithOSPath(
+	filename *accessors.OSPath) (
 	accessors.FileInfo, error) {
 
 	fullpath := path_specs.FromGenericComponentList(filename.Components)
@@ -218,7 +219,7 @@ func (self FileStoreFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPat
 	var fullpath api.FSPathSpec
 
 	// It is a data store path
-	if filename.Components[0] == "ds:" {
+	if filename.PathSpec().DelegatePath == "ds:" {
 		ds_path := getDSPathSpec(filename)
 		fullpath = ds_path.AsFilestorePath()
 		switch ds_path.Type() {
@@ -228,6 +229,7 @@ func (self FileStoreFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPat
 		case api.PATH_TYPE_DATASTORE_PROTO:
 			fullpath = fullpath.SetType(api.PATH_TYPE_FILESTORE_DB)
 		}
+
 	} else {
 		fullpath = path_specs.FromGenericComponentList(filename.Components)
 	}

accessors/vql_arg_parser.go

@@ -69,7 +69,13 @@ func ParseOSPath(ctx context.Context,
 			last_idx := len(components) - 1
 			components[last_idx] += api.GetExtensionForFilestore(t)
 		}
-		return MustNewFileStorePath("fs:").Append(components...), nil
+		res := MustNewFileStorePath("fs:").Append(components...)
+
+		// Store the FSPathSpec in the data for fast retrieval if we
+		// are passed to the fs accessor (this is commonly the case).
+		res.Data = t
+
+		return res, nil
 
 	case api.DSPathSpec:
 		// Create an OSPath to represent the abstract filestore path.
@@ -82,11 +88,6 @@ func ParseOSPath(ctx context.Context,
 		}
 		return MustNewFileStorePath("ds:").Append(components...), nil
 
-		// WHERE version(plugin="glob") > 2:
-		// Initializer can be a list of components. In this case we
-		// take the base pathspec (which is accessor determined) and
-		// add the components to it.
-
 	case string:
 		return accessor.ParsePath(t)
 

api/download.go

@@ -495,8 +495,7 @@ func getTransformer(
 func downloadFileStore(prefix []string) http.Handler {
 	return api_utils.HandlerFunc(nil,
 		func(w http.ResponseWriter, r *http.Request) {
-			path_spec := paths.FSPathSpecFromClientPath(r.URL.Path)
-			components := path_spec.Components()
+			components := utils.SplitComponents(r.URL.Path)
 
 			// make sure the prefix is correct
 			for i, p := range prefix {
@@ -506,6 +505,8 @@ func downloadFileStore(prefix []string) http.Handler {
 				}
 			}
 
+			path_spec := path_specs.FromGenericComponentList(components)
+
 			org_id := authenticators.GetOrgIdFromRequest(r)
 			org_manager, err := services.GetOrgManager()
 			if err != nil {

artifacts/testdata/server/testcases/artifacts.out.yaml

@@ -41,6 +41,7 @@ Output: [
   "started": "2019-11-08 07:30:59.920512962 +0000 UTC",
   "vfs_path": "fs:/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/old_style.zip",
   "expected_size": 1319,
+  "client_path": "",
   "Upload": {
    "Path": "/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/old_style.zip",
    "Size": 0,

file_store/directory/directory.go

@@ -37,6 +37,7 @@ import (
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/datastore"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
+	"www.velocidex.com/golang/velociraptor/file_store/path_specs"
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/third_party/cache"
 	"www.velocidex.com/golang/velociraptor/utils"
@@ -89,6 +90,8 @@ func (self *DirectoryFileStore) ListDirectory(dirname api.FSPathSpec) (
 		return nil, err
 	}
 
+	untyped := path_specs.IsComponentUntyped(dirname.Components())
+
 	var result []api.FileInfo
 	for _, fileinfo := range files {
 		// Each file from the filesystem will be potentially
@@ -100,19 +103,26 @@ func (self *DirectoryFileStore) ListDirectory(dirname api.FSPathSpec) (
 			continue
 		}
 
+		// Name may be compressed
+		name = datastore.UncompressComponent(
+			self.db, self.config_obj, name)
+
+		// Fixme: Use api.FromGenericComponentList
 		var name_type api.PathType
 		if fileinfo.IsDir() {
 			name_type = api.PATH_TYPE_DATASTORE_DIRECTORY
+
+		} else if untyped {
+			name_type = api.PATH_TYPE_FILESTORE_ANY
+
 		} else {
 			name_type, name = api.GetFileStorePathTypeFromExtension(name)
 		}
-		result = append(result, file_store_file_info.NewFileStoreFileInfo(
-			self.config_obj,
-			dirname.AddUnsafeChild(
-				datastore.UncompressComponent(
-					self.db, self.config_obj, name)).
-				SetType(name_type),
-			fileinfo))
+
+		result = append(result,
+			file_store_file_info.NewFileStoreFileInfo(self.config_obj,
+				dirname.AddUnsafeChild(name).SetType(name_type),
+				fileinfo))
 	}
 
 	return result, nil
@@ -152,6 +162,7 @@ func (self *DirectoryFileStore) ReadFile(
 	chunk_file_path := datastore.AsFilestoreFilename(
 		self.db, self.config_obj, filename.
 			SetType(api.PATH_TYPE_FILESTORE_CHUNK_INDEX))
+
 	chunk_fd, err := os.OpenFile(chunk_file_path, os.O_RDWR, 0600)
 	if err != nil {
 		return reader, nil

file_store/memory/memory.go

@@ -209,6 +209,8 @@ func (self *MemoryFileStore) ListDirectory(root_path api.FSPathSpec) ([]api.File
 
 	root_components := root_path.Components()
 
+	untyped := path_specs.IsComponentUntyped(root_components)
+
 	// Mapping between the base name and the files
 	seen_files := make(map[string]api.FileInfo)
 	seen_dirs := make(map[string]api.FileInfo)
@@ -245,8 +247,15 @@ func (self *MemoryFileStore) ListDirectory(root_path api.FSPathSpec) ([]api.File
 				continue
 			}
 
+			name := path_spec.Base()
+			// Force the file to be untyped.
+			if untyped {
+				name += api.GetExtensionForFilestore(path_spec)
+				path_spec = path_spec.SetType(api.PATH_TYPE_FILESTORE_ANY)
+			}
+
 			new_child := &vtesting.MockFileInfo{
-				Name_:     path_spec.Base(),
+				Name_:     name,
 				PathSpec_: path_spec,
 				FullPath_: path_spec.AsClientPath(),
 				Size_:     int64(len(v)),

file_store/path_specs/utils.go

@@ -44,17 +44,88 @@ func AsGenericComponentList(path api.FSPathSpec) []string {
 	return components
 }
 
-// Builds a filestore pathspec from a plain components list. Uses the
-// extension of the filename component to determine the path type.
+// Builds a filestore pathspec from a plain components list.
+//
+// A PathSpec contains a list of components **and** a type, so just a
+// list of components is not sufficient to infer the type. This
+// function relied on internal knowledge of the filestore structure to
+// infer the correct type from the component list.
 func FromGenericComponentList(components []string) api.FSPathSpec {
-	pathspec := NewUnsafeFilestorePath(components...)
-	if len(components) > 0 {
-		last_idx := len(components) - 1
-		fs_type, name := api.GetFileStorePathTypeFromExtension(
-			components[last_idx])
-		return pathspec.Dir().AddChild(name).SetType(fs_type)
+	components, path_type := getTypeFromComponents(components)
+	return NewUnsafeFilestorePath(components...).SetType(path_type)
+}
+
+var (
+	anyPrefixes = [][]string{
+		[]string{"public"},
+		[]string{"backups"},
+		[]string{"temp"},
+
+		// Uploaded collections from the client.
+		[]string{"clients", "", "collections", "", "uploads"},
+
+		// Notebooks: attachments and uploads
+		[]string{"notebooks", "", "attach"},
+		[]string{"notebooks", "", "", "uploads"},
+
+		// Client notebooks
+		[]string{"clients", "", "collections", "", "notebook", "", "attach"},
+		[]string{"clients", "", "collections", "", "notebook", "", "", "uploads"},
+
+		// Client monitoring notebooks
+		[]string{"clients", "", "monitoring_notebooks", "", "attach"},
+		[]string{"clients", "", "monitoring_notebooks", "", "", "uploads"},
+
+		// Hunt notebooks
+		[]string{"hunts", "", "notebook", "", "attach"},
+		[]string{"hunts", "", "notebook", "", "", "uploads"},
 	}
-	return pathspec
+)
+
+// Returns true if the components address a path which is untyped.
+func IsComponentUntyped(components []string) bool {
+	return MatchComponentPattern(components, anyPrefixes)
+}
+
+func MatchComponentPattern(components []string, patterns [][]string) bool {
+	// Everything under the public path is untyped.
+	for _, prefix := range patterns {
+		if matchPrefix(components, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func getTypeFromComponents(components []string) ([]string, api.PathType) {
+	if len(components) == 0 || IsComponentUntyped(components) {
+		return components, api.PATH_TYPE_FILESTORE_ANY
+	}
+
+	// Client uploads are all untyped
+	if len(components) > 4 && components[0] == "clients" {
+		return components, api.PATH_TYPE_FILESTORE_ANY
+	}
+
+	last_component := components[len(components)-1]
+
+	// Fallback, use the extension to deduce the type.
+	fs_type, name := api.GetFileStorePathTypeFromExtension(last_component)
+	return append(components[:len(components)-1], name), fs_type
+}
+
+func matchPrefix(components []string, prefix []string) bool {
+	if len(components) < len(prefix) {
+		return false
+	}
+
+	for idx, m := range prefix {
+		if m != "" && components[idx] != m {
+			return false
+		}
+	}
+
+	return true
 }
 
 // Converts a typed pathspec to an untyped pathspec. This is required

file_store/path_specs/utils_test.go

@@ -0,0 +1,94 @@
+package path_specs
+
+import (
+	"testing"
+
+	"www.velocidex.com/golang/velociraptor/file_store/api"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
+)
+
+type testCaseT struct {
+	components  []string
+	client_path string
+	path_type   api.PathType
+}
+
+func TestFromGenericComponentList(t *testing.T) {
+	for _, tc := range []testCaseT{
+		{
+			components: []string{"downloads",
+				"server", "F.D1CDBN7O9PTU4",
+				"server-server-F.D1CDBN7O9PTU4.zip"},
+			client_path: "/downloads/server/F.D1CDBN7O9PTU4/server-server-F.D1CDBN7O9PTU4.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_DOWNLOAD_ZIP,
+		},
+		{
+			// Public directory is always PATH_TYPE_FILESTORE_ANY
+			components:  []string{"public", "test.zip"},
+			client_path: "/public/test.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_ANY,
+		},
+
+		{
+			// Global notebook attachments are always PATH_TYPE_FILESTORE_ANY
+			components: []string{
+				"notebooks", "N.D55OJV2COB544", "attach", "NA.D56I70FP35OKU-file.zip"},
+			client_path: "/notebooks/N.D55OJV2COB544/attach/NA.D56I70FP35OKU-file.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_ANY,
+		},
+		{
+			// Global notebook uploads are always PATH_TYPE_FILESTORE_ANY
+			components: []string{
+				"notebooks", "N.D55OJV2COB544", "NC.D56I6S6LK00FI-D56I71V0BJULE",
+				"uploads", "data", "file.zip"},
+			client_path: "/notebooks/N.D55OJV2COB544/NC.D56I6S6LK00FI-D56I71V0BJULE/uploads/data/file.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_ANY,
+		},
+		{
+			// Client notebook attachments are always PATH_TYPE_FILESTORE_ANY
+			components: []string{
+				"clients", "C.d7f8859f5e0e01f7", "collections", "F.D55T34A0NIDTC",
+				"notebook", "N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7", "attach",
+				"NA.D56HQ0GRL4CK0-file.zip"},
+			client_path: "/clients/C.d7f8859f5e0e01f7/collections/F.D55T34A0NIDTC/notebook/N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7/attach/NA.D56HQ0GRL4CK0-file.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_ANY,
+		},
+		{
+			// Client notebook uploads are always PATH_TYPE_FILESTORE_ANY
+			components: []string{
+				"clients", "C.d7f8859f5e0e01f7", "collections", "F.D55T34A0NIDTC",
+				"notebook", "N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7",
+				"NC.D56CPJR8RE9GO-D56HRHRS9QR6A", "uploads", "file",
+				"file.zip"},
+			client_path: "/clients/C.d7f8859f5e0e01f7/collections/F.D55T34A0NIDTC/notebook/N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7/NC.D56CPJR8RE9GO-D56HRHRS9QR6A/uploads/file/file.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_ANY,
+		},
+
+		{
+			// Hunt notebook attachments are always PATH_TYPE_FILESTORE_ANY
+			components: []string{
+				"hunts", "H.D4ASRT5R531G4", "notebook", "N.H.D4ASRT5R531G4",
+				"attach", "NA.D56HQ0GRL4CK0-file.zip"},
+			client_path: "/hunts/H.D4ASRT5R531G4/notebook/N.H.D4ASRT5R531G4/attach/NA.D56HQ0GRL4CK0-file.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_ANY,
+		},
+		{
+			// Client notebook uploads are always PATH_TYPE_FILESTORE_ANY
+			components: []string{
+				"hunts", "H.D4ASRT5R531G4", "notebook", "N.H.D4ASRT5R531G4",
+				"NC.D4ASRVHMIJFK0-D56I6463RE2QU", "uploads", "data",
+				"file.zip"},
+			client_path: "/hunts/H.D4ASRT5R531G4/notebook/N.H.D4ASRT5R531G4/NC.D4ASRVHMIJFK0-D56I6463RE2QU/uploads/data/file.zip",
+			path_type:   api.PATH_TYPE_FILESTORE_ANY,
+		},
+	} {
+		path := FromGenericComponentList(tc.components)
+		assert.Equal(t, path.Type(), tc.path_type,
+			"PathType is not correct: %v vs %v (%v)",
+			path.Type(), tc.path_type, tc.components)
+
+		assert.Equal(t, path.AsClientPath(), tc.client_path,
+			"ClietPath is not correct: %v vs %v",
+			path.AsClientPath(), tc.client_path)
+	}
+}