feat: enable importing docker compose module (#2327)

## Description
Enables importing and running Docker Compose modules from Starlark:

```
djangopg = import_module("github.com/kurtosis-tech/django-compose/docker-compose.yml")
flaskredis = import_module("github.com/docker/awesome-compose/flask-redis/compose.yaml")

def run(plan, args):
    djangopg.run(plan)
    flaskredis.run(plan)
```

## Is this change user facing?
YES

core/server/api_container/main.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/interpretation_time_value_store"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider"
 	"net"
 	"os"
 	"path"
@@ -126,10 +127,11 @@ func runMain() error {
 		return stacktrace.Propagate(err, "An error occurred while getting the enclave db")
 	}
 
-	gitPackageContentProvider, err := enclaveDataDir.GetGitPackageContentProvider(enclaveDb)
+	repositoriesDirPath, tempDirectoriesDirPath, githubAuthTokenFilePath, err := enclaveDataDir.GetEnclaveDataDirectoryPaths()
 	if err != nil {
-		return stacktrace.Propagate(err, "An error occurred while creating the Git module content provider")
+		return stacktrace.Propagate(err, "An error occurred getting directory paths of the enclave data directory.")
 	}
+	gitPackageContentProvider := git_package_content_provider.NewGitPackageContentProvider(repositoriesDirPath, tempDirectoriesDirPath, githubAuthTokenFilePath, enclaveDb)
 
 	// TODO Extract into own function
 	var kurtosisBackend backend_interface.KurtosisBackend

core/server/api_container/server/api_container_service.go

@@ -12,8 +12,8 @@ import (
 	"compress/gzip"
 	"context"
 	"fmt"
-	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/docker_compose_transpiler"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/docker_compose_transpiler"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_structure"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/instructions_plan/resolver"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml"
@@ -76,15 +76,6 @@ const (
 	defaultParallelism       = 4
 )
 
-var defaultComposeFilenames = []string{
-	"compose.yml",
-	"compose.yaml",
-	"docker-compose.yml",
-	"docker-compose.yaml",
-	"docker_compose.yml",
-	"docker_compose.yaml",
-}
-
 // Guaranteed (by a unit test) to be a 1:1 mapping between API port protos and port spec protos
 var apiContainerPortProtoToPortSpecPortProto = map[kurtosis_core_rpc_api_bindings.Port_TransportProtocol]port_spec.TransportProtocol{
 	kurtosis_core_rpc_api_bindings.Port_TCP:  port_spec.TransportProtocol_TCP,
@@ -887,7 +878,7 @@ func (apicService *ApiContainerService) runStarlarkPackageSetup(
 
 	// If kurtosis.yml doesn't exist, assume a Compose package and transpile compose into starlark
 	if relativePathToMainFile == "" {
-		for _, defaultComposeFilename := range defaultComposeFilenames {
+		for _, defaultComposeFilename := range docker_compose_transpiler.DefaultComposeFilenames {
 			candidateComposeAbsFilepath := path.Join(packageRootPathOnDisk, defaultComposeFilename)
 			if _, err := os.Stat(candidateComposeAbsFilepath); err == nil {
 				relativePathToMainFile = defaultComposeFilename
@@ -900,7 +891,7 @@ func (apicService *ApiContainerService) runStarlarkPackageSetup(
 					"default Compose files (%s) were found. Either add a '%s' file to the package root or add one of the "+
 					"default Compose files.",
 				startosis_constants.KurtosisYamlName,
-				strings.Join(defaultComposeFilenames, ", "),
+				strings.Join(docker_compose_transpiler.DefaultComposeFilenames, ", "),
 				startosis_constants.KurtosisYamlName,
 			)
 		}

core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go → core/server/api_container/server/startosis_engine/docker_compose_transpiler/docker_compose_transpiler.go

@@ -72,6 +72,15 @@ var (
 	possibleHttpPorts               = []uint32{8080, 8000, 80, 443}
 )
 
+var DefaultComposeFilenames = []string{
+	"compose.yml",
+	"compose.yaml",
+	"docker-compose.yml",
+	"docker-compose.yaml",
+	"docker_compose.yml",
+	"docker_compose.yaml",
+}
+
 type ComposeService types.ServiceConfig
 
 type StarlarkServiceConfig *kurtosis_type_constructor.KurtosisValueTypeDefault

core/server/api_container/server/startosis_engine/kurtosis_types/service_config/image_build_spec.go

@@ -9,7 +9,6 @@ import (
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_type_constructor"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types"
-	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
 	"go.starlark.net/starlark"
@@ -221,10 +220,6 @@ func getOnDiskImageBuildSpecPaths(
 	locatorOfModuleInWhichThisBuiltInIsBeingCalled string,
 	packageContentProvider startosis_packages.PackageContentProvider,
 	packageReplaceOptions map[string]string) (string, string, *startosis_errors.InterpretationError) {
-	if packageId == startosis_constants.PackageIdPlaceholderForStandaloneScript {
-		return "", "", startosis_errors.NewInterpretationError("Cannot use ImageBuildSpec in a standalone script; create a package and rerun to use ImageBuildSpec.")
-	}
-
 	// get absolute locator of context directory
 	contextDirAbsoluteLocator, interpretationErr := packageContentProvider.GetAbsoluteLocator(packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, buildContextLocator, packageReplaceOptions)
 	if interpretationErr != nil {

core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go

@@ -9,6 +9,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/docker_compose_transpiler"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
@@ -20,6 +21,7 @@ import (
 	"io/fs"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 )
 
@@ -40,7 +42,7 @@ const (
 	// this gets us the entire history - useful for fetching commits on a repo
 	depthAssumingBranchTagsCommitsAreSpecified = 0
 
-	filePathToKurtosisYamlNotFound           = ""
+	filePathToKurtosisOrComposeYamlNotFound  = ""
 	replaceCountPackageDirWithGithubConstant = 1
 
 	osPathSeparatorString = string(os.PathSeparator)
@@ -131,7 +133,7 @@ func (provider *GitPackageContentProvider) getOnDiskAbsolutePath(absoluteLocator
 	}
 
 	// Check if the repo exists
-	// If the repo exists but the `pathToFileOnDisk` doesn't that means there's a mistake in the locator
+	// If the repo exists but the `pathToFileOnDisk` doesn't exist, the locator is invalid
 	if _, err := os.Stat(pathToPackageOnDisk); err == nil {
 		relativeFilePathWithoutPackageName := strings.Replace(parsedURL.GetRelativeFilePath(), parsedURL.GetRelativeRepoPath(), replacedWithEmptyString, onlyOneReplacement)
 		return "", startosis_errors.NewInterpretationError("'%v' doesn't exist in the package '%v'", relativeFilePathWithoutPackageName, parsedURL.GetRelativeRepoPath())
@@ -147,17 +149,19 @@ func (provider *GitPackageContentProvider) getOnDiskAbsolutePath(absoluteLocator
 	}
 
 	// check whether kurtosis yaml exists in the path
-	maybeKurtosisYamlPath, interpretationError := getKurtosisYamlPathForFileUrl(pathToFileOnDisk, provider.repositoriesDir)
+	maybeKurtosisOrComposeYamlPath, interpretationError := getKurtosisOrComposeYamlPathForFile(pathToFileOnDisk, provider.repositoriesDir)
 	if interpretationError != nil {
 		return "", startosis_errors.WrapWithInterpretationError(err, "Error occurred while verifying whether '%v' belongs to a Kurtosis package.", repositoryPathURL)
 	}
 
-	if maybeKurtosisYamlPath == filePathToKurtosisYamlNotFound {
-		return "", startosis_errors.NewInterpretationError("%v is not found in the path of '%v'; files can only be accessed from Kurtosis packages. For more information, go to: %v", startosis_constants.KurtosisYamlName, repositoryPathURL, user_support_constants.HowImportWorksLink)
+	if maybeKurtosisOrComposeYamlPath == filePathToKurtosisOrComposeYamlNotFound {
+		return "", startosis_errors.NewInterpretationError("%v or valid Docker Compose yaml not found in the path of '%v'; files can only be accessed from Kurtosis packages. For more information, go to: %v", startosis_constants.KurtosisYamlName, repositoryPathURL, user_support_constants.HowImportWorksLink)
 	}
 
-	if _, interpretationError = validateAndGetKurtosisYaml(maybeKurtosisYamlPath, provider.repositoriesDir); interpretationError != nil {
-		return "", interpretationError
+	if containsKurtosisYaml(maybeKurtosisOrComposeYamlPath) {
+		if _, interpretationError = validateAndGetKurtosisYaml(maybeKurtosisOrComposeYamlPath, provider.repositoriesDir); interpretationError != nil {
+			return "", interpretationError
+		}
 	}
 
 	return pathToFileOnDisk, nil
@@ -169,13 +173,20 @@ func (provider *GitPackageContentProvider) GetModuleContents(absoluteLocator *st
 		return "", interpretationError
 	}
 
-	// Load the file content from its absolute path
-	contents, err := os.ReadFile(pathToFile)
-	if err != nil {
-		return "", startosis_errors.WrapWithInterpretationError(err, "Loading module content for module '%s' failed. An error occurred in reading contents of the file '%v'", absoluteLocator.GetLocator(), pathToFile)
+	// if pathToFile contains compose yaml, assume Docker Compose Package
+	if containsComposeYaml(pathToFile) {
+		contents, err := docker_compose_transpiler.TranspileDockerComposePackageToStarlark(filepath.Dir(pathToFile), filepath.Base(pathToFile))
+		if err != nil {
+			return "", startosis_errors.WrapWithInterpretationError(err, "Loading module content for module '%s' failed. An error occurred in transpiling the Docker Compose Package to Starlark at path '%v'", absoluteLocator.GetLocator(), pathToFile)
+		}
+		return contents, nil
+	} else {
+		contentsBytes, err := os.ReadFile(pathToFile)
+		if err != nil {
+			return "", startosis_errors.WrapWithInterpretationError(err, "Loading module content for module '%s' failed. An error occurred in reading contents of the file '%v'", absoluteLocator.GetLocator(), pathToFile)
+		}
+		return string(contentsBytes), nil
 	}
-
-	return string(contents), nil
 }
 
 func (provider *GitPackageContentProvider) GetOnDiskAbsolutePackagePath(packageId string) (string, *startosis_errors.InterpretationError) {
@@ -500,7 +511,7 @@ func validateAndGetKurtosisYaml(absPathToKurtosisYmlInThePackage string, package
 		return nil, startosis_errors.WrapWithInterpretationError(errWhileParsing, "Error occurred while parsing %v", absPathToKurtosisYmlInThePackage)
 	}
 
-	// this method validates whether the package name is also the locator - it should the location where kurtosis.yml exists
+	// this method validates whether the package name is also the locator - it should be the location where kurtosis.yml exists
 	if err := validatePackageNameMatchesKurtosisYamlLocation(kurtosisYaml, absPathToKurtosisYmlInThePackage, packageDir); err != nil {
 		return nil, startosis_errors.WrapWithInterpretationError(err, "Error occurred while validating %v", absPathToKurtosisYmlInThePackage)
 	}
@@ -538,43 +549,49 @@ func validatePackageNameMatchesKurtosisYamlLocation(kurtosisYaml *yaml_parser.Ku
 // TODO: we should clean this up and have a dependency management system; all the dependencies should be stated kurtosis.yml upfront
 // TODO: this will simplify our validation process, and enable customers to use local packages like go.
 // TODO: in my opinion - we should eventually clone and validate the packages even before we start the interpretation process, maybe inside
-//
-//	api_container_service
-func getKurtosisYamlPathForFileUrl(absPathToFile string, packagesDir string) (string, *startosis_errors.InterpretationError) {
-	return getKurtosisYamlPathForFileUrlInternal(absPathToFile, packagesDir, os.Stat)
+// TODO: api_container_service
+func getKurtosisOrComposeYamlPathForFile(absPathToFile string, packagesDir string) (string, *startosis_errors.InterpretationError) {
+	return getKurtosisOrComposeYamlPathForFileUrlInternal(absPathToFile, packagesDir, os.Stat)
 }
 
-// This method walks along the path of the file and determines whether kurtosis.yml is found in any directory. If the path is found, it returns
-// the absolute path of kurtosis.yml, otherwise it returns an empty string when the kurtosis.yml is not found.
+// This method walks along the path of the file and determines whether kurtosis.yml or a valid compose.yml is found in any directory. If the path is found, it returns
+// the absolute path of the .yml, otherwise it returns an empty string when either kurtosis.yml or valid compose.yml not found.
 //
 // For example, the path to the file is /kurtosis-data/startosis-packages/some-repo/some-folder/some-file-to-be-read.star
 // This method will start the walk from some-repo, then go to some-folder and so on.
 // It will continue the search for kurtosis.yml until either kurtosis.yml is found or the path is fully transversed.
-func getKurtosisYamlPathForFileUrlInternal(absPathToFile string, packagesDir string, stat func(string) (os.FileInfo, error)) (string, *startosis_errors.InterpretationError) {
+func getKurtosisOrComposeYamlPathForFileUrlInternal(absPathToFile string, packagesDir string, stat func(string) (os.FileInfo, error)) (string, *startosis_errors.InterpretationError) {
 	// it will remove /kurtosis-data/startosis-package from absPathToFile and start the search from repo itself.
 	// we can be sure that kurtosis.yml will never be found in those folders.
 	beginSearchForKurtosisYamlFromRepo := strings.TrimPrefix(absPathToFile, packagesDir)
 	if beginSearchForKurtosisYamlFromRepo == absPathToFile {
-		return filePathToKurtosisYamlNotFound, startosis_errors.NewInterpretationError("Absolute path to file: %v must start with following prefix %v", absPathToFile, packagesDir)
+		return filePathToKurtosisOrComposeYamlNotFound, startosis_errors.NewInterpretationError("Absolute path to file: %v must start with following prefix %v", absPathToFile, packagesDir)
 	}
 
 	removeTrailingPathSeparator := strings.Trim(beginSearchForKurtosisYamlFromRepo, osPathSeparatorString)
 	dirs := strings.Split(removeTrailingPathSeparator, osPathSeparatorString)
 	logrus.Debugf("Found directories: %v", dirs)
 
+	var validYamlFilenames []string
+	validYamlFilenames = append(validYamlFilenames, startosis_constants.KurtosisYamlName)
+	validYamlFilenames = append(validYamlFilenames, docker_compose_transpiler.DefaultComposeFilenames...)
+
 	maybePackageRootPath := packagesDir
 	for _, dir := range dirs[:len(dirs)-1] {
 		maybePackageRootPath = path.Join(maybePackageRootPath, dir)
-		pathToKurtosisYaml := path.Join(maybePackageRootPath, startosis_constants.KurtosisYamlName)
-		if _, err := stat(pathToKurtosisYaml); err == nil {
-			logrus.Debugf("Found root path: %v", maybePackageRootPath)
-			// the method should return the absolute path to minimize the confusion
-			return pathToKurtosisYaml, nil
-		} else if !errors.Is(err, os.ErrNotExist) {
-			return filePathToKurtosisYamlNotFound, startosis_errors.WrapWithInterpretationError(err, "An error occurred while locating %v in the path of '%v'", startosis_constants.KurtosisYamlName, absPathToFile)
+		for _, validYamlFilename := range validYamlFilenames {
+			pathToYaml := path.Join(maybePackageRootPath, validYamlFilename)
+			if _, err := stat(pathToYaml); err == nil {
+				logrus.Debugf("Found root path: %v", maybePackageRootPath)
+				// the method should return the absolute path to minimize the confusion
+				return pathToYaml, nil
+			} else if !errors.Is(err, os.ErrNotExist) {
+				return filePathToKurtosisOrComposeYamlNotFound, startosis_errors.WrapWithInterpretationError(err, "An error occurred while locating %v in the path of '%v'", validYamlFilename, absPathToFile)
+			}
 		}
 	}
-	return filePathToKurtosisYamlNotFound, nil
+
+	return filePathToKurtosisOrComposeYamlNotFound, nil
 }
 
 func isLocalDependencyReplace(replace string) bool {
@@ -583,3 +600,17 @@ func isLocalDependencyReplace(replace string) bool {
 	}
 	return false
 }
+
+// Returns if kurtosisOrComposeYamlPath contains the kurtosis.yml substring
+func containsKurtosisYaml(kurtosisOrComposeYamlPath string) bool {
+	return strings.Contains(kurtosisOrComposeYamlPath, startosis_constants.KurtosisYamlName)
+}
+
+func containsComposeYaml(filepath string) bool {
+	for _, composeYaml := range docker_compose_transpiler.DefaultComposeFilenames {
+		if strings.Contains(filepath, composeYaml) {
+			return true
+		}
+	}
+	return false
+}