diff --git a/Makefile b/Makefile index fad10b7a..7a88cce0 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,9 @@ subo/docker/publish: subo/smoketest: subo ./scripts/smoketest.sh +subo/toolchaintest: subo subo/docker + ./scripts/toolchaintest.sh + mod/replace/atmo: go mod edit -replace github.com/suborbital/atmo=$(HOME)/Workspaces/suborbital/atmo diff --git a/builder/builder.go b/builder/builder.go index 3b3b3ff6..87563af8 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -93,7 +93,7 @@ func (b *Builder) BuildWithToolchain(tcn Toolchain) error { } if tcn == ToolchainNative { - b.log.LogStart(fmt.Sprintf("building runnable: %s (%s)", mod.Name, mod.Module.Lang)) + b.log.LogStart(fmt.Sprintf("building module: %s (%s)", mod.Name, mod.Module.Lang)) result := &BuildResult{} @@ -157,7 +157,18 @@ func (b *Builder) dockerBuildForLang(lang string) (*BuildResult, error) { result := &BuildResult{} - outputLog, err := b.Config.CommandRunner.Run(fmt.Sprintf("docker run --rm --mount type=bind,source=%s,target=/root/runnable %s subo build %s --native --langs %s", b.Context.MountPath, img, b.Context.RelDockerPath, lang)) + // Run the Docker toolchain as the current user + uid := os.Getuid() + gid := os.Getgid() + + toolchainCmd := fmt.Sprintf("docker run --rm --mount type=bind,source=%s,target=/usr/src/runnable -u %d:%d --env HOME=/tmp %s subo build %s --native --langs %s", b.Context.MountPath, uid, gid, img, b.Context.RelDockerPath, lang) + + // Show the toolchain command being run + if b.Context.LogLevel == "verbose" { + util.LogInfo(fmt.Sprintf("🐳 %s", toolchainCmd)) + } + + outputLog, err := b.Config.CommandRunner.Run(toolchainCmd) result.OutputLog = outputLog diff --git a/builder/docker/assemblyscript/Dockerfile b/builder/docker/assemblyscript/Dockerfile index 2b2d00b6..64adbafe 100644 --- a/builder/docker/assemblyscript/Dockerfile +++ b/builder/docker/assemblyscript/Dockerfile @@ -1,6 +1,6 @@ FROM suborbital/subo:dev as subo FROM node:16-buster-slim -WORKDIR /root/runnable +WORKDIR /usr/src/runnable COPY --from=subo /go/bin/subo /usr/local/bin RUN npm install -g npm@latest diff --git a/builder/docker/grain/Dockerfile b/builder/docker/grain/Dockerfile index 711d571c..7cd2a9ee 100644 --- a/builder/docker/grain/Dockerfile +++ b/builder/docker/grain/Dockerfile @@ -1,6 +1,6 @@ FROM suborbital/subo:dev as subo FROM ghcr.io/grain-lang/grain:0.4-slim -WORKDIR /root/runnable +WORKDIR /usr/src/runnable COPY --from=subo /go/bin/subo /usr/local/bin/subo RUN mkdir /root/suborbital diff --git a/builder/docker/javascript/Dockerfile b/builder/docker/javascript/Dockerfile index b36e7bc4..bb6fcfdb 100644 --- a/builder/docker/javascript/Dockerfile +++ b/builder/docker/javascript/Dockerfile @@ -2,11 +2,10 @@ FROM ghcr.io/suborbital/javy:v0.3.0 as javy FROM suborbital/subo:dev as subo FROM node:16-bullseye-slim -WORKDIR /root/runnable +WORKDIR /usr/src/runnable # Propagate our root permissions for our home folder to everyone. This allows # npm scripts (which get run as whatever user owns the mounted runnable # directory) to access common home folder resources (caches, etc.). -RUN mkdir /root/suborbital && chmod -R o=u /root COPY --from=javy /usr/local/bin/javy /usr/local/bin COPY --from=subo /go/bin/subo /usr/local/bin diff --git a/builder/docker/rust/Dockerfile b/builder/docker/rust/Dockerfile index bc410a41..23b79b8a 100644 --- a/builder/docker/rust/Dockerfile +++ b/builder/docker/rust/Dockerfile @@ -1,11 +1,14 @@ FROM suborbital/subo:dev as subo FROM rust:1.56.1-slim-buster -WORKDIR /root/runnable -COPY --from=subo /go/bin/subo /usr/local/bin +WORKDIR /usr/src/runnable + +# install the wasm target and then force an update of the crates.io index +RUN rustup target install wasm32-wasi && \ + cargo search suborbital -# install the wasm target and then install something that -# doesn't exist (and ignore the error) to update the crates.io index -RUN mkdir /root/suborbital && \ - rustup target install wasm32-wasi -RUN cargo install lazy_static; exit 0 +# make everything inside CARGO_HOME writable for anyone (again) +RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +# embed subo +COPY --from=subo /go/bin/subo /usr/local/bin diff --git a/builder/docker/swift/Dockerfile b/builder/docker/swift/Dockerfile index 4fb50cae..d79384b2 100644 --- a/builder/docker/swift/Dockerfile +++ b/builder/docker/swift/Dockerfile @@ -1,5 +1,5 @@ FROM suborbital/subo:dev as subo FROM ghcr.io/swiftwasm/swift:focal -WORKDIR /root/runnable +WORKDIR /usr/src/runnable COPY --from=subo /go/bin/subo /usr/local/bin diff --git a/builder/docker/tinygo/Dockerfile b/builder/docker/tinygo/Dockerfile index da259c87..ce9255dc 100644 --- a/builder/docker/tinygo/Dockerfile +++ b/builder/docker/tinygo/Dockerfile @@ -18,7 +18,7 @@ RUN wget -O tinygo.tar.gz \ rm -rf tinygo/src/examples && \ rm -rf tinygo.tar.gz -WORKDIR /root/runnable +WORKDIR /usr/src/runnable COPY --from=go /usr/local/go /usr/local/ COPY --from=subo /go/bin/subo /usr/local/bin diff --git a/builder/prereq.go b/builder/prereq.go index 82138bee..225abad5 100644 --- a/builder/prereq.go +++ b/builder/prereq.go @@ -115,7 +115,7 @@ func (p Prereq) GetCommand(b BuildConfig, md project.ModuleDir) (string, error) var fullCmd strings.Builder err = cmdTmpl.Execute(&fullCmd, data) if err != nil { - return "", errors.Wrap(err, "failed to execute prerequisite Command string with runnableDir") + return "", errors.Wrap(err, "failed to execute prerequisite Command string with moduleDir") } return fullCmd.String(), nil diff --git a/project/context.go b/project/context.go index ae4e61d1..c1dcea87 100644 --- a/project/context.go +++ b/project/context.go @@ -30,16 +30,17 @@ var validLangs = map[string]struct{}{ // Context describes the context under which the tool is being run. type Context struct { - Cwd string - CwdIsRunnable bool - Modules []ModuleDir - Bundle BundleRef - TenantConfig *tenant.Config - DeltavVersion string - Langs []string - MountPath string - RelDockerPath string - BuilderTag string + Cwd string + CwdIsRunnable bool + Modules []ModuleDir + Bundle BundleRef + TenantConfig *tenant.Config + RuntimeVersion string + Langs []string + MountPath string + RelDockerPath string + BuilderTag string + LogLevel string } // ModuleDir represents a directory containing a Runnable. @@ -66,7 +67,7 @@ func ForDirectory(dir string) (*Context, error) { modules, cwdIsRunnable, err := getModuleDirs(fullDir) if err != nil { - return nil, errors.Wrap(err, "failed to getRunnableDirs") + return nil, errors.Wrap(err, "failed to getModuleDirs") } bundle, err := bundleTargetPath(fullDir) @@ -189,11 +190,11 @@ func getModuleDirs(cwd string) ([]ModuleDir, bool, error) { return nil, false, errors.Wrap(err, "failed to list directory") } - // Check to see if we're running from within a Runnable directory + // Check to see if we're running from within a Module directory // and return true if so. moduleDir, err := getModuleFromFiles(cwd, topLvlFiles) if err != nil { - return nil, false, errors.Wrap(err, "failed to getRunnableFromFiles") + return nil, false, errors.Wrap(err, "failed to getModuleFromFiles") } else if moduleDir != nil { modules = append(modules, *moduleDir) return modules, true, nil @@ -215,7 +216,7 @@ func getModuleDirs(cwd string) ([]ModuleDir, bool, error) { moduleDir, err := getModuleFromFiles(dirPath, innerFiles) if err != nil { - return nil, false, errors.Wrap(err, "failed to getRunnableFromFiles") + return nil, false, errors.Wrap(err, "failed to getModuleFromFiles") } else if moduleDir == nil { continue } diff --git a/root.go b/root.go index e543f115..423f5672 100644 --- a/root.go +++ b/root.go @@ -26,8 +26,8 @@ including building WebAssembly Runnables and Atmo projects.`, // create commands. create := &cobra.Command{ Use: "create", - Short: "create a runnable, project, or handler", - Long: `create a new Atmo project, WebAssembly runnable or handler`, + Short: "create a module, project, or handler", + Long: `create a new E2Core project, WebAssembly module or handler`, } if features.EnableReleaseCommands { diff --git a/scripts/smoketest.sh b/scripts/smoketest.sh index 6f9dfea1..0755cf93 100755 --- a/scripts/smoketest.sh +++ b/scripts/smoketest.sh @@ -27,11 +27,11 @@ subo create project "$TEST_PROJECT" pushd "$TEST_PROJECT" > /dev/null # create a runnable for each supported language -subo create runnable rs-test --lang rust -subo create runnable swift-test --lang swift -subo create runnable as-test --lang assemblyscript -subo create runnable tinygo-test --lang tinygo -subo create runnable js-test --lang javascript +subo create module rs-test --lang rust +subo create module swift-test --lang swift +subo create module as-test --lang assemblyscript +subo create module tinygo-test --lang tinygo +subo create module js-test --lang javascript # build project bundle subo build . \ No newline at end of file diff --git a/scripts/toolchaintest.sh b/scripts/toolchaintest.sh new file mode 100755 index 00000000..b5591375 --- /dev/null +++ b/scripts/toolchaintest.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -eu + +TEST_PROJECT="buildertest" + +trap 'catch $? $LINENO' EXIT + +catch() { + if [[ "$1" != "0" ]]; then + echo "An Error $1 occurred on $2" + fi + + # return to origin, clear directory stack + pushd -0 > /dev/null && dirs -c + + if [[ -d "$TEST_PROJECT" ]]; then + echo "Cleaning up test artifacts..." + rm -rf "$TEST_PROJECT" || echo "Failed to clean up test artifacts, if this was a permissions error try using 'sudo rm -rf $TEST_PROJECT'" + fi +} + +# build local subo docker image +docker image ls -q suborbital/subo:dev +if [[ "$?" != "0" ]]; then + make subo/docker +fi + +# rebuild local docker build tooling +docker build . -f builder/docker/assemblyscript/Dockerfile -t suborbital/builder-as:dev +docker build . -f builder/docker/grain/Dockerfile --platform linux/amd64 -t suborbital/builder-gr:dev +docker build . -f builder/docker/javascript/Dockerfile -t suborbital/builder-js:dev +docker build . -f builder/docker/rust/Dockerfile -t suborbital/builder-rs:dev +docker build . -f builder/docker/swift/Dockerfile -t suborbital/builder-swift:dev +docker build . -f builder/docker/tinygo/Dockerfile --platform linux/amd64 --build-arg TARGETARCH=amd64 -t suborbital/builder-tinygo:dev +docker build . -f builder/docker/wat/Dockerfile -t suborbital/builder-wat:dev + + +# create a new project +subo create project "$TEST_PROJECT" + +# enter project directory +pushd "$TEST_PROJECT" > /dev/null + +# create a runnable for each supported language +subo create module rs-test --lang rust +subo create module swift-test --lang swift +subo create module as-test --lang assemblyscript +subo create module tinygo-test --lang tinygo +subo create module js-test --lang javascript + +# build project bundle +subo build . --builder-tag dev --verbose diff --git a/subo/command/build.go b/subo/command/build.go index f5e4e647..c931d9f1 100644 --- a/subo/command/build.go +++ b/subo/command/build.go @@ -15,8 +15,8 @@ import ( func BuildCmd() *cobra.Command { cmd := &cobra.Command{ Use: "build [dir]", - Short: "build a WebAssembly runnable", - Long: `build a WebAssembly runnable and/or create a Runnable Bundle`, + Short: "build a WebAssembly module", + Long: `build a WebAssembly module and/or create an application bundle`, RunE: func(cmd *cobra.Command, args []string) error { dir := "." if len(args) > 0 { @@ -29,11 +29,11 @@ func BuildCmd() *cobra.Command { } if len(bdr.Context.Modules) == 0 { - return errors.New("🚫 no runnables found in current directory (no .runnable.yaml files found)") + return errors.New(fmt.Sprintf("🚫 build found no modules in %s (no .module.yaml files found)", bdr.Context.Cwd)) } if bdr.Context.CwdIsRunnable { - util.LogInfo("building single Runnable (run from project root to create bundle)") + util.LogInfo("building a single module (run from project root to create bundle)") } langs, _ := cmd.Flags().GetStringSlice("langs") @@ -44,7 +44,7 @@ func BuildCmd() *cobra.Command { shouldDockerBuild, _ := cmd.Flags().GetBool("docker") if bdr.Context.CwdIsRunnable && shouldDockerBuild { - return errors.New("🚫 cannot build Docker image for a single Runnable (must be a project)") + return errors.New("🚫 cannot build Docker image for a single module (must be a project)") } useNative, _ := cmd.Flags().GetBool("native") @@ -69,6 +69,11 @@ func BuildCmd() *cobra.Command { bdr.Context.BuilderTag = builderTag } + verbose, _ := cmd.Flags().GetBool("verbose") + if verbose { + bdr.Context.LogLevel = "verbose" + } + if makeTarget != "" { util.LogStart(fmt.Sprintf("make %s", makeTarget)) _, err = util.Command.Run(fmt.Sprintf("make %s", makeTarget)) @@ -113,10 +118,13 @@ func BuildCmd() *cobra.Command { cmd.Flags().Bool("native", false, "use native (locally installed) toolchain rather than Docker") cmd.Flags().String("make", "", "execute the provided Make target before building the project bundle") cmd.Flags().Bool("docker", false, "build your project's Dockerfile. It will be tagged {identifier}:{appVersion}") - cmd.Flags().StringSlice("langs", []string{}, "build only Runnables for the listed languages (comma-seperated)") + cmd.Flags().StringSlice("langs", []string{}, "build only modules for the listed languages (comma-seperated)") cmd.Flags().String("mountpath", "", "if passed, the Docker builders will mount their volumes at the provided path") cmd.Flags().String("relpath", "", "if passed, the Docker builders will run `subo build` using the provided path, relative to '--mountpath'") cmd.Flags().String("builder-tag", "", "use the provided tag for builder images") + cmd.Flags().BoolP("verbose", "v", false, "enable verbose logging") + cmd.Flags().Lookup("verbose").NoOptDefVal = "true" + return cmd } diff --git a/subo/command/clean.go b/subo/command/clean.go index 34b9fc0f..e8b2f5c7 100644 --- a/subo/command/clean.go +++ b/subo/command/clean.go @@ -14,7 +14,7 @@ import ( "github.com/suborbital/subo/subo/util" ) -//CleanCmd removes all of the target/.build folders for Runnables and deletes the .wasm files. +// CleanCmd removes all of the target/.build folders for Runnables and deletes the .wasm files. func CleanCmd() *cobra.Command { cmd := &cobra.Command{ Use: "clean", @@ -32,7 +32,7 @@ func CleanCmd() *cobra.Command { } if len(bctx.Modules) == 0 { - return errors.New("🚫 no runnables found in current directory (no .runnable yaml files found)") + return errors.New("🚫 cleanup found no modules in current directory (no .module.yaml files found)") } util.LogStart(fmt.Sprintf("cleaning in %s", bctx.Cwd)) diff --git a/subo/command/create_project.go b/subo/command/create_project.go index b7e6a875..aad55bec 100644 --- a/subo/command/create_project.go +++ b/subo/command/create_project.go @@ -19,10 +19,10 @@ const ( ) type projectData struct { - Name string - Environment string - APIVersion string - DeltavVersion string + Name string + Environment string + APIVersion string + RuntimeVersion string } // CreateProjectCmd returns the build command. @@ -68,10 +68,10 @@ func CreateProjectCmd() *cobra.Command { } data := projectData{ - Name: name, - Environment: environment, - APIVersion: release.FFIVersion, - DeltavVersion: release.DeltavVersion, + Name: name, + Environment: environment, + APIVersion: release.FFIVersion, + RuntimeVersion: release.RuntimeVersion, } if err := template.ExecTmplDir(bctx.Cwd, name, templatesPath, "project", data); err != nil { diff --git a/subo/command/dev.go b/subo/command/dev.go index 246db754..99aa42e8 100644 --- a/subo/command/dev.go +++ b/subo/command/dev.go @@ -43,7 +43,7 @@ func DevCmd() *cobra.Command { util.LogInfo("Running DeltaV with debug logging") } - dockerCmd := fmt.Sprintf("docker run -v=%s:/home/atmo -e=DELTAV_HTTP_PORT=%s %s -p=%s:%s suborbital/deltav:%s deltav start", bctx.Cwd, port, envvar, port, port, release.DeltavVersion) + dockerCmd := fmt.Sprintf("docker run -v=%s:/home/atmo -e=DELTAV_HTTP_PORT=%s %s -p=%s:%s suborbital/deltav:%s deltav start", bctx.Cwd, port, envvar, port, port, release.RuntimeVersion) _, err = util.Command.Run(dockerCmd) if err != nil { diff --git a/subo/release/version.go b/subo/release/version.go index dafdc99f..f40a2569 100644 --- a/subo/release/version.go +++ b/subo/release/version.go @@ -7,8 +7,8 @@ var SuboDotVersion = "0.5.4" // FFIVersion is the FFI version used by this version of subo. var FFIVersion = "0.15.1" -// DeltavVersion is the default version of Deltav that will be used for new projects. -var DeltavVersion = "0.4.7" +// RuntimeVersion is the default version of Deltav that will be used for new projects. +var RuntimeVersion = "0.4.7" // SCCTag is the docker tag used for creating new compute core deployments. var SCCTag = "v0.3.1"