diff --git a/accessors/api.go b/accessors/api.go index ebfe3c07f17..c9885fc2d56 100644 --- a/accessors/api.go +++ b/accessors/api.go @@ -54,6 +54,11 @@ type OSPath struct { pathspec *PathSpec serialized *string Manipulator PathManipulator + + // Opaque data that can be stored in the OSPath. This provides a + // mechanism to transport additional data in the OSPath and avoid + // having to convert back and forth. + Data interface{} } func (self *OSPath) Equal(other *OSPath) bool { diff --git a/accessors/file_store/accessor.go b/accessors/file_store/accessor.go index f94a0b2ded8..27c47305244 100644 --- a/accessors/file_store/accessor.go +++ b/accessors/file_store/accessor.go @@ -4,7 +4,6 @@ package file_store // the generic filestore. This allows us to run globs on the file // store regardless of the specific filestore implementation. import ( - "encoding/json" "errors" "www.velocidex.com/golang/velociraptor/accessors" @@ -14,6 +13,7 @@ import ( "www.velocidex.com/golang/velociraptor/file_store" "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/file_store/path_specs" + "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/uploads" "www.velocidex.com/golang/velociraptor/utils" "www.velocidex.com/golang/velociraptor/utils/files" @@ -103,7 +103,8 @@ func (self FileStoreFileSystemAccessor) Lstat(filename string) ( return self.LstatWithOSPath(full_path) } -func (self FileStoreFileSystemAccessor) LstatWithOSPath(filename *accessors.OSPath) ( +func (self FileStoreFileSystemAccessor) LstatWithOSPath( + filename *accessors.OSPath) ( accessors.FileInfo, error) { fullpath := path_specs.FromGenericComponentList(filename.Components) @@ -218,7 +219,7 @@ func (self FileStoreFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPat var fullpath api.FSPathSpec // It is a data store path - if filename.Components[0] == "ds:" { + if filename.PathSpec().DelegatePath == "ds:" { ds_path := getDSPathSpec(filename) fullpath = ds_path.AsFilestorePath() switch ds_path.Type() { @@ -228,6 +229,7 @@ func (self FileStoreFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPat case api.PATH_TYPE_DATASTORE_PROTO: fullpath = fullpath.SetType(api.PATH_TYPE_FILESTORE_DB) } + } else { fullpath = path_specs.FromGenericComponentList(filename.Components) } diff --git a/accessors/vql_arg_parser.go b/accessors/vql_arg_parser.go index 40dd11111d9..c860df6e609 100644 --- a/accessors/vql_arg_parser.go +++ b/accessors/vql_arg_parser.go @@ -69,7 +69,13 @@ func ParseOSPath(ctx context.Context, last_idx := len(components) - 1 components[last_idx] += api.GetExtensionForFilestore(t) } - return MustNewFileStorePath("fs:").Append(components...), nil + res := MustNewFileStorePath("fs:").Append(components...) + + // Store the FSPathSpec in the data for fast retrieval if we + // are passed to the fs accessor (this is commonly the case). + res.Data = t + + return res, nil case api.DSPathSpec: // Create an OSPath to represent the abstract filestore path. @@ -82,11 +88,6 @@ func ParseOSPath(ctx context.Context, } return MustNewFileStorePath("ds:").Append(components...), nil - // WHERE version(plugin="glob") > 2: - // Initializer can be a list of components. In this case we - // take the base pathspec (which is accessor determined) and - // add the components to it. - case string: return accessor.ParsePath(t) diff --git a/api/download.go b/api/download.go index 98129b03816..72e6abcc441 100644 --- a/api/download.go +++ b/api/download.go @@ -495,8 +495,7 @@ func getTransformer( func downloadFileStore(prefix []string) http.Handler { return api_utils.HandlerFunc(nil, func(w http.ResponseWriter, r *http.Request) { - path_spec := paths.FSPathSpecFromClientPath(r.URL.Path) - components := path_spec.Components() + components := utils.SplitComponents(r.URL.Path) // make sure the prefix is correct for i, p := range prefix { @@ -506,6 +505,8 @@ func downloadFileStore(prefix []string) http.Handler { } } + path_spec := path_specs.FromGenericComponentList(components) + org_id := authenticators.GetOrgIdFromRequest(r) org_manager, err := services.GetOrgManager() if err != nil { diff --git a/artifacts/testdata/server/testcases/artifacts.out.yaml b/artifacts/testdata/server/testcases/artifacts.out.yaml index 2f05954bfe6..1a8eec824b3 100644 --- a/artifacts/testdata/server/testcases/artifacts.out.yaml +++ b/artifacts/testdata/server/testcases/artifacts.out.yaml @@ -41,6 +41,7 @@ Output: [ "started": "2019-11-08 07:30:59.920512962 +0000 UTC", "vfs_path": "fs:/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/old_style.zip", "expected_size": 1319, + "client_path": "", "Upload": { "Path": "/clients/C.4f5e52adf0a337a9/collections/F.BN2HJCPOF5U7U/uploads/file/C:/old_style.zip", "Size": 0, diff --git a/file_store/directory/directory.go b/file_store/directory/directory.go index e26e668cabe..1f8cc6bfff4 100644 --- a/file_store/directory/directory.go +++ b/file_store/directory/directory.go @@ -37,6 +37,7 @@ import ( config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/file_store/path_specs" logging "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/third_party/cache" "www.velocidex.com/golang/velociraptor/utils" @@ -89,6 +90,8 @@ func (self *DirectoryFileStore) ListDirectory(dirname api.FSPathSpec) ( return nil, err } + untyped := path_specs.IsComponentUntyped(dirname.Components()) + var result []api.FileInfo for _, fileinfo := range files { // Each file from the filesystem will be potentially @@ -100,19 +103,26 @@ func (self *DirectoryFileStore) ListDirectory(dirname api.FSPathSpec) ( continue } + // Name may be compressed + name = datastore.UncompressComponent( + self.db, self.config_obj, name) + + // Fixme: Use api.FromGenericComponentList var name_type api.PathType if fileinfo.IsDir() { name_type = api.PATH_TYPE_DATASTORE_DIRECTORY + + } else if untyped { + name_type = api.PATH_TYPE_FILESTORE_ANY + } else { name_type, name = api.GetFileStorePathTypeFromExtension(name) } - result = append(result, file_store_file_info.NewFileStoreFileInfo( - self.config_obj, - dirname.AddUnsafeChild( - datastore.UncompressComponent( - self.db, self.config_obj, name)). - SetType(name_type), - fileinfo)) + + result = append(result, + file_store_file_info.NewFileStoreFileInfo(self.config_obj, + dirname.AddUnsafeChild(name).SetType(name_type), + fileinfo)) } return result, nil @@ -152,6 +162,7 @@ func (self *DirectoryFileStore) ReadFile( chunk_file_path := datastore.AsFilestoreFilename( self.db, self.config_obj, filename. SetType(api.PATH_TYPE_FILESTORE_CHUNK_INDEX)) + chunk_fd, err := os.OpenFile(chunk_file_path, os.O_RDWR, 0600) if err != nil { return reader, nil diff --git a/file_store/memory/memory.go b/file_store/memory/memory.go index 2670a0824bb..40a55591e30 100644 --- a/file_store/memory/memory.go +++ b/file_store/memory/memory.go @@ -209,6 +209,8 @@ func (self *MemoryFileStore) ListDirectory(root_path api.FSPathSpec) ([]api.File root_components := root_path.Components() + untyped := path_specs.IsComponentUntyped(root_components) + // Mapping between the base name and the files seen_files := make(map[string]api.FileInfo) seen_dirs := make(map[string]api.FileInfo) @@ -245,8 +247,15 @@ func (self *MemoryFileStore) ListDirectory(root_path api.FSPathSpec) ([]api.File continue } + name := path_spec.Base() + // Force the file to be untyped. + if untyped { + name += api.GetExtensionForFilestore(path_spec) + path_spec = path_spec.SetType(api.PATH_TYPE_FILESTORE_ANY) + } + new_child := &vtesting.MockFileInfo{ - Name_: path_spec.Base(), + Name_: name, PathSpec_: path_spec, FullPath_: path_spec.AsClientPath(), Size_: int64(len(v)), diff --git a/file_store/path_specs/utils.go b/file_store/path_specs/utils.go index e48ef7f17d3..d46e9cf2796 100644 --- a/file_store/path_specs/utils.go +++ b/file_store/path_specs/utils.go @@ -44,17 +44,88 @@ func AsGenericComponentList(path api.FSPathSpec) []string { return components } -// Builds a filestore pathspec from a plain components list. Uses the -// extension of the filename component to determine the path type. +// Builds a filestore pathspec from a plain components list. +// +// A PathSpec contains a list of components **and** a type, so just a +// list of components is not sufficient to infer the type. This +// function relied on internal knowledge of the filestore structure to +// infer the correct type from the component list. func FromGenericComponentList(components []string) api.FSPathSpec { - pathspec := NewUnsafeFilestorePath(components...) - if len(components) > 0 { - last_idx := len(components) - 1 - fs_type, name := api.GetFileStorePathTypeFromExtension( - components[last_idx]) - return pathspec.Dir().AddChild(name).SetType(fs_type) + components, path_type := getTypeFromComponents(components) + return NewUnsafeFilestorePath(components...).SetType(path_type) +} + +var ( + anyPrefixes = [][]string{ + []string{"public"}, + []string{"backups"}, + []string{"temp"}, + + // Uploaded collections from the client. + []string{"clients", "", "collections", "", "uploads"}, + + // Notebooks: attachments and uploads + []string{"notebooks", "", "attach"}, + []string{"notebooks", "", "", "uploads"}, + + // Client notebooks + []string{"clients", "", "collections", "", "notebook", "", "attach"}, + []string{"clients", "", "collections", "", "notebook", "", "", "uploads"}, + + // Client monitoring notebooks + []string{"clients", "", "monitoring_notebooks", "", "attach"}, + []string{"clients", "", "monitoring_notebooks", "", "", "uploads"}, + + // Hunt notebooks + []string{"hunts", "", "notebook", "", "attach"}, + []string{"hunts", "", "notebook", "", "", "uploads"}, } - return pathspec +) + +// Returns true if the components address a path which is untyped. +func IsComponentUntyped(components []string) bool { + return MatchComponentPattern(components, anyPrefixes) +} + +func MatchComponentPattern(components []string, patterns [][]string) bool { + // Everything under the public path is untyped. + for _, prefix := range patterns { + if matchPrefix(components, prefix) { + return true + } + } + return false +} + +func getTypeFromComponents(components []string) ([]string, api.PathType) { + if len(components) == 0 || IsComponentUntyped(components) { + return components, api.PATH_TYPE_FILESTORE_ANY + } + + // Client uploads are all untyped + if len(components) > 4 && components[0] == "clients" { + return components, api.PATH_TYPE_FILESTORE_ANY + } + + last_component := components[len(components)-1] + + // Fallback, use the extension to deduce the type. + fs_type, name := api.GetFileStorePathTypeFromExtension(last_component) + return append(components[:len(components)-1], name), fs_type +} + +func matchPrefix(components []string, prefix []string) bool { + if len(components) < len(prefix) { + return false + } + + for idx, m := range prefix { + if m != "" && components[idx] != m { + return false + } + } + + return true } // Converts a typed pathspec to an untyped pathspec. This is required diff --git a/file_store/path_specs/utils_test.go b/file_store/path_specs/utils_test.go new file mode 100644 index 00000000000..6d4cb801a63 --- /dev/null +++ b/file_store/path_specs/utils_test.go @@ -0,0 +1,94 @@ +package path_specs + +import ( + "testing" + + "www.velocidex.com/golang/velociraptor/file_store/api" + "www.velocidex.com/golang/velociraptor/vtesting/assert" +) + +type testCaseT struct { + components []string + client_path string + path_type api.PathType +} + +func TestFromGenericComponentList(t *testing.T) { + for _, tc := range []testCaseT{ + { + components: []string{"downloads", + "server", "F.D1CDBN7O9PTU4", + "server-server-F.D1CDBN7O9PTU4.zip"}, + client_path: "/downloads/server/F.D1CDBN7O9PTU4/server-server-F.D1CDBN7O9PTU4.zip", + path_type: api.PATH_TYPE_FILESTORE_DOWNLOAD_ZIP, + }, + { + // Public directory is always PATH_TYPE_FILESTORE_ANY + components: []string{"public", "test.zip"}, + client_path: "/public/test.zip", + path_type: api.PATH_TYPE_FILESTORE_ANY, + }, + + { + // Global notebook attachments are always PATH_TYPE_FILESTORE_ANY + components: []string{ + "notebooks", "N.D55OJV2COB544", "attach", "NA.D56I70FP35OKU-file.zip"}, + client_path: "/notebooks/N.D55OJV2COB544/attach/NA.D56I70FP35OKU-file.zip", + path_type: api.PATH_TYPE_FILESTORE_ANY, + }, + { + // Global notebook uploads are always PATH_TYPE_FILESTORE_ANY + components: []string{ + "notebooks", "N.D55OJV2COB544", "NC.D56I6S6LK00FI-D56I71V0BJULE", + "uploads", "data", "file.zip"}, + client_path: "/notebooks/N.D55OJV2COB544/NC.D56I6S6LK00FI-D56I71V0BJULE/uploads/data/file.zip", + path_type: api.PATH_TYPE_FILESTORE_ANY, + }, + { + // Client notebook attachments are always PATH_TYPE_FILESTORE_ANY + components: []string{ + "clients", "C.d7f8859f5e0e01f7", "collections", "F.D55T34A0NIDTC", + "notebook", "N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7", "attach", + "NA.D56HQ0GRL4CK0-file.zip"}, + client_path: "/clients/C.d7f8859f5e0e01f7/collections/F.D55T34A0NIDTC/notebook/N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7/attach/NA.D56HQ0GRL4CK0-file.zip", + path_type: api.PATH_TYPE_FILESTORE_ANY, + }, + { + // Client notebook uploads are always PATH_TYPE_FILESTORE_ANY + components: []string{ + "clients", "C.d7f8859f5e0e01f7", "collections", "F.D55T34A0NIDTC", + "notebook", "N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7", + "NC.D56CPJR8RE9GO-D56HRHRS9QR6A", "uploads", "file", + "file.zip"}, + client_path: "/clients/C.d7f8859f5e0e01f7/collections/F.D55T34A0NIDTC/notebook/N.F.D55T34A0NIDTC-C.d7f8859f5e0e01f7/NC.D56CPJR8RE9GO-D56HRHRS9QR6A/uploads/file/file.zip", + path_type: api.PATH_TYPE_FILESTORE_ANY, + }, + + { + // Hunt notebook attachments are always PATH_TYPE_FILESTORE_ANY + components: []string{ + "hunts", "H.D4ASRT5R531G4", "notebook", "N.H.D4ASRT5R531G4", + "attach", "NA.D56HQ0GRL4CK0-file.zip"}, + client_path: "/hunts/H.D4ASRT5R531G4/notebook/N.H.D4ASRT5R531G4/attach/NA.D56HQ0GRL4CK0-file.zip", + path_type: api.PATH_TYPE_FILESTORE_ANY, + }, + { + // Client notebook uploads are always PATH_TYPE_FILESTORE_ANY + components: []string{ + "hunts", "H.D4ASRT5R531G4", "notebook", "N.H.D4ASRT5R531G4", + "NC.D4ASRVHMIJFK0-D56I6463RE2QU", "uploads", "data", + "file.zip"}, + client_path: "/hunts/H.D4ASRT5R531G4/notebook/N.H.D4ASRT5R531G4/NC.D4ASRVHMIJFK0-D56I6463RE2QU/uploads/data/file.zip", + path_type: api.PATH_TYPE_FILESTORE_ANY, + }, + } { + path := FromGenericComponentList(tc.components) + assert.Equal(t, path.Type(), tc.path_type, + "PathType is not correct: %v vs %v (%v)", + path.Type(), tc.path_type, tc.components) + + assert.Equal(t, path.AsClientPath(), tc.client_path, + "ClietPath is not correct: %v vs %v", + path.AsClientPath(), tc.client_path) + } +} diff --git a/file_store/tests/testsuite.go b/file_store/tests/testsuite.go index a6ace496a0e..c19ab5b913c 100644 --- a/file_store/tests/testsuite.go +++ b/file_store/tests/testsuite.go @@ -111,13 +111,6 @@ func (self *FileStoreTestSuite) TestListChildrenSameNameDifferentTypes() { assert.NoError(self.T(), err) fd.Close() - // FIXME: This will create a file named Foo in place of the - // intermediate directory causing the below to fail. - //fd, err = self.filestore.WriteFile(dir_path_spec.AddChild("Foo"). - // SetType(api.PATH_TYPE_FILESTORE_ANY)) - //assert.NoError(self.T(), err) - //fd.Close() - // Add an intermediate directory - this will add a directory info // for the intermediate directory. fd, err = self.filestore.WriteFile(dir_path_spec.AddChild("Foo", "dir", "value"). @@ -155,7 +148,9 @@ func (self *FileStoreTestSuite) TestListChildrenSameNameDifferentTypes() { json.MustMarshalIndent(golden)) } -// List children recovers child's type based on extensions. +// List children recovers child's type based on extensions. NOTE: +// This only works in typed directories. For Untyped directories, +// ListDirectory() always recovers types as untyped. func (self *FileStoreTestSuite) TestListChildrenWithTypes() { for idx, t := range []api.PathType{ @@ -217,6 +212,51 @@ func (self *FileStoreTestSuite) TestListChildrenWithTypes() { } } +// Storing files in untypes locations recovers them as untyped. +func (self *FileStoreTestSuite) TestListChildrenUntypedPaths() { + + for idx, t := range []api.PathType{ + api.PATH_TYPE_FILESTORE_JSON_INDEX, + api.PATH_TYPE_FILESTORE_JSON, + api.PATH_TYPE_FILESTORE_JSON_TIME_INDEX, + + // Used to write sparse indexes + api.PATH_TYPE_FILESTORE_SPARSE_IDX, + + // Used to write zip files in the download folder. + api.PATH_TYPE_FILESTORE_DOWNLOAD_ZIP, + api.PATH_TYPE_FILESTORE_DOWNLOAD_REPORT, + + // TMP files + api.PATH_TYPE_FILESTORE_TMP, + api.PATH_TYPE_FILESTORE_CSV, + + // Used for artifacts + api.PATH_TYPE_FILESTORE_YAML, + + api.PATH_TYPE_FILESTORE_ANY, + } { + filename := path_specs.NewSafeFilestorePath( + "public", fmt.Sprintf("b%v", idx)).SetType(t) + + fd, err := self.filestore.WriteFile(filename.AddChild("Foo.txt")) + assert.NoError(self.T(), err) + fd.Close() + + infos, err := self.filestore.ListDirectory(filename) + assert.NoError(self.T(), err) + + assert.Equal(self.T(), 1, len(infos)) + + // Upon reading the path should be untyped. + assert.Equal(self.T(), api.PATH_TYPE_FILESTORE_ANY, infos[0].PathSpec().Type()) + + // The filename should contain the extension as part of the file. + assert.Equal(self.T(), infos[0].Name(), "Foo.txt"+ + api.GetExtensionForFilestore(filename)) + } +} + func (self *FileStoreTestSuite) TestListDirectory() { filename := path_specs.NewSafeFilestorePath("a", "b") fd, err := self.filestore.WriteFile(filename.AddChild("Foo.txt")) diff --git a/paths/utils.go b/paths/utils.go index f6fb6006df2..50147e220dd 100644 --- a/paths/utils.go +++ b/paths/utils.go @@ -20,13 +20,5 @@ func DSPathSpecFromClientPath(client_path string) api.DSPathSpec { func FSPathSpecFromClientPath(client_path string) api.FSPathSpec { components := ExtractClientPathComponents(client_path) - result := path_specs.NewUnsafeFilestorePath(components...) - if len(components) > 0 { - last := len(components) - 1 - name_type, name := api.GetFileStorePathTypeFromExtension( - components[last]) - components[last] = name - return result.SetType(name_type) - } - return result + return path_specs.FromGenericComponentList(components) } diff --git a/server/comms.go b/server/comms.go index b4f9322ef59..898a100c751 100644 --- a/server/comms.go +++ b/server/comms.go @@ -36,7 +36,7 @@ import ( "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/file_store" "www.velocidex.com/golang/velociraptor/file_store/api" - "www.velocidex.com/golang/velociraptor/paths" + "www.velocidex.com/golang/velociraptor/file_store/path_specs" "www.velocidex.com/golang/velociraptor/services" "www.velocidex.com/golang/velociraptor/utils" @@ -156,7 +156,11 @@ func PrepareFrontendMux( // from the filestore. router.Handle(base+"/public/", GetLoggingHandler(config_obj, "/public")( http.StripPrefix(base, - downloadPublic(config_obj, []string{"public"})))) + downloadPublic(config_obj, [][]string{ + // Allow all files in the public directory to be + // accessible. + []string{"public"}, + })))) return nil } @@ -742,21 +746,22 @@ func GetLoggingHandler(config_obj *config_proto.Config, } } +// A handler that makes parts of the file store available for +// download. This is used to directly download e.g. attachments in +// notebooks. func downloadPublic( - config_obj *config_proto.Config, prefix []string) http.Handler { + config_obj *config_proto.Config, patterns [][]string) http.Handler { return api_utils.HandlerFunc(nil, func(w http.ResponseWriter, r *http.Request) { - path_spec := paths.FSPathSpecFromClientPath(r.URL.Path) - components := path_spec.Components() + components := utils.SplitComponents(r.URL.Path) // make sure the prefix is correct - for i, p := range prefix { - if len(components) <= i || p != components[i] { - returnError(config_obj, w, 404, notFoundError) - return - } + if !path_specs.MatchComponentPattern(components, patterns) { + returnError(config_obj, w, 404, notFoundError) + return } + path_spec := path_specs.FromGenericComponentList(components) file_store_factory := file_store.GetFileStore(config_obj) fd, err := file_store_factory.ReadFile(path_spec) if err != nil { diff --git a/vql/server/flows/uploads.go b/vql/server/flows/uploads.go index f7a6a774016..5966065da4b 100644 --- a/vql/server/flows/uploads.go +++ b/vql/server/flows/uploads.go @@ -111,6 +111,7 @@ func (self UploadsPlugins) Call( Set("file_size", upload.Size). Set("uploaded_size", upload.Size). Set("vfs_path", vfs_path). + Set("client_path", ""). Set("Upload", uploads.UploadResponse{ Path: vfs_path.String(), Size: upload.Size, @@ -227,6 +228,8 @@ func readFlowUploads( pathspec = path_specs.NewUnsafeFilestorePath( utils.SplitComponents(vfs_path)...). SetType(api.PATH_TYPE_FILESTORE_ANY) + + row.Set("client_path", "") } row.Update("vfs_path", pathspec)