Skip to content
This repository has been archived by the owner on May 4, 2021. It is now read-only.

Commit

Permalink
Add push command (#302)
Browse files Browse the repository at this point in the history
* Add push command

Co-authored-by: Yiran Wang <[email protected]>
  • Loading branch information
sema and yiranwang52 authored Feb 6, 2020
1 parent c75663b commit 9ed3b61
Show file tree
Hide file tree
Showing 18 changed files with 579 additions and 77 deletions.
Binary file removed .DS_Store
Binary file not shown.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ env
__pycache__
*.pyc
.vscode
.DS_Store
6 changes: 3 additions & 3 deletions bin/makisu/cmd/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,9 @@ type buildCmd struct {
func getBuildCmd() *buildCmd {
buildCmd := &buildCmd{
Command: &cobra.Command{
Use: "build -t=<image_tag> [flags] <context_path>",
Use: "build -t=<image_tag> [flags] <context_path>",
DisableFlagsInUseLine: true,
Short: "Build docker image, optionally push to registries and/or load into docker daemon",
Short: "Build docker image, optionally push to registries and/or load into docker daemon",
},
}
buildCmd.Args = func(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -151,7 +151,7 @@ func (cmd *buildCmd) processFlags() error {
return fmt.Errorf("invalid commit option: %s", cmd.commit)
}

if err := cmd.initRegistryConfig(); err != nil {
if err := initRegistryConfig(cmd.registryConfig); err != nil {
return fmt.Errorf("failed to initialize registry configuration: %s", err)
}

Expand Down
265 changes: 265 additions & 0 deletions bin/makisu/cmd/push.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,265 @@
// Copyright (c) 2018 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"

"github.com/uber/makisu/lib/docker/image"
"github.com/uber/makisu/lib/log"
"github.com/uber/makisu/lib/registry"
"github.com/uber/makisu/lib/storage"
"github.com/uber/makisu/lib/tario"
"github.com/uber/makisu/lib/utils"

"github.com/spf13/cobra"
)

type pushCmd struct {
*cobra.Command

tag string

pushRegistries []string
replicas []string
registryConfig string
}

func getPushCmd() *pushCmd {
pushCmd := &pushCmd{
Command: &cobra.Command{
Use: "push -t=<image_tag> [flags] <image_tar_path>",
DisableFlagsInUseLine: true,
Short: "Push docker image to registries",
},
}
pushCmd.Args = func(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
return errors.New("Requires image tar path as argument")
}
return nil
}
pushCmd.Run = func(cmd *cobra.Command, args []string) {
if err := pushCmd.processFlags(); err != nil {
log.Errorf("failed to process flags: %s", err)
os.Exit(1)
}

if err := pushCmd.Push(args[0]); err != nil {
log.Error(err)
os.Exit(1)
}
}

pushCmd.PersistentFlags().StringVarP(&pushCmd.tag, "tag", "t", "", "Image tag (required)")

pushCmd.PersistentFlags().StringArrayVar(&pushCmd.pushRegistries, "push", nil, "Registry to push image to")
pushCmd.PersistentFlags().StringArrayVar(&pushCmd.replicas, "replica", nil, "Push targets with alternative full image names \"<registry>/<repo>:<tag>\"")
pushCmd.PersistentFlags().StringVar(&pushCmd.registryConfig, "registry-config", "", "Set build-time variables")

pushCmd.MarkFlagRequired("tag")
pushCmd.Flags().SortFlags = false
pushCmd.PersistentFlags().SortFlags = false

return pushCmd
}

func (cmd *pushCmd) processFlags() error {
if err := initRegistryConfig(cmd.registryConfig); err != nil {
return fmt.Errorf("failed to initialize registry configuration: %s", err)
}

return nil
}

// Push image tar to docker registries.
func (cmd *pushCmd) Push(imageTarPath string) error {
log.Infof("Starting Makisu push (version=%s)", utils.BuildHash)

imageName, err := cmd.getTargetImageName()
if err != nil {
return err
}

// TODO: make configurable?
store, err := storage.NewImageStore("/tmp/makisu-storage")
if err != nil {
return fmt.Errorf("unable to create internal store: %s", err)
}

if err := cmd.loadImageTarIntoStore(store, imageName, cmd.replicas, imageTarPath); err != nil {
return fmt.Errorf("unable to import image: %s", err)
}

// Push image to registries that were specified in the --push flag.
for _, registry := range cmd.pushRegistries {
target := imageName.WithRegistry(registry)
if err := cmd.pushImage(store, target); err != nil {
return fmt.Errorf("failed to push image: %s", err)
}
}
for _, replica := range cmd.replicas {
target := image.MustParseName(replica)
if err := cmd.pushImage(store, target); err != nil {
return fmt.Errorf("failed to push image: %s", err)
}
}

log.Infof("Finished pushing %s", imageName.ShortName())
return nil
}

func (cmd *pushCmd) getTargetImageName() (image.Name, error) {
if cmd.tag == "" {
msg := "please specify a target image name: push -t=<image_tag> [flags] <image_tar_path>"
return image.Name{}, errors.New(msg)
}

return image.MustParseName(cmd.tag), nil
}

func (cmd *pushCmd) loadImageTarIntoStore(
store *storage.ImageStore, imageName image.Name, replicas []string, imageTarPath string) error {

if err := cmd.importTar(store, imageName, replicas, imageTarPath); err != nil {
return fmt.Errorf("import image tar: %s", err)
}

return nil
}

func (cmd *pushCmd) pushImage(store *storage.ImageStore, imageName image.Name) error {
registryClient := registry.New(store, imageName.GetRegistry(), imageName.GetRepository())
if err := registryClient.Push(imageName.GetTag()); err != nil {
return fmt.Errorf("failed to push image: %s", err)
}
log.Infof("Successfully pushed %s to %s", imageName, imageName.GetRegistry())
return nil
}

// importTar imports an image, as a tar, to the image store.
func (cmd *pushCmd) importTar(
store *storage.ImageStore, imageName image.Name, replicas []string, tarPath string) error {

repo, tag := imageName.GetRepository(), imageName.GetTag()

// Extract tar into temporary directory.
dir := filepath.Join(store.SandboxDir, repo, tag)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("create unpack directory: %s", err)
}
defer os.RemoveAll(dir)

reader, err := os.Open(tarPath)
if err != nil {
return fmt.Errorf("open tar file: %s", err)
}
defer reader.Close()

if err := tario.Untar(reader, dir); err != nil {
return fmt.Errorf("unpack tar: %s", err)
}

// Read manifest.
exportManifestPath := filepath.Join(dir, "manifest.json")
exportManifestData, err := ioutil.ReadFile(exportManifestPath)
if err != nil {
return fmt.Errorf("read export manifest: %s", err)
}

var exportManifests []image.ExportManifest
if err := json.Unmarshal(exportManifestData, &exportManifests); err != nil {
return fmt.Errorf("unmarshal export manifest: %s", err)
}

for _, exportManifest := range exportManifests {
// Import extracted dir content into image store -- {sha}.json.
configPath := filepath.Join(dir, exportManifest.Config.String())

configInfo, err := os.Stat(configPath)
if err != nil {
return fmt.Errorf("lookup config file info: %s", err)
}

configReader, err := os.Open(configPath)
if err != nil {
return fmt.Errorf("open config json: %s", err)
}
defer configReader.Close()
configDigest, err := image.NewDigester().FromReader(configReader)

if err := store.Layers.LinkStoreFileFrom(
configDigest.Hex(), configPath); err != nil && !os.IsExist(err) {

return fmt.Errorf("commit config to store: %s", err)
}

// Import extracted dir content into image store -- {sha}/layer.tar.
var layers []image.Descriptor
for _, layer := range exportManifest.Layers {
layerPath := path.Join(dir, layer.String())

layerInfo, err := os.Stat(layerPath)
if err != nil {
return fmt.Errorf("lookup layer file info: %s", err)
}

layerReader, err := os.Open(layerPath)
if err != nil {
return fmt.Errorf("open layer tar: %s", err)
}
defer layerReader.Close()
layerDigest, err := image.NewDigester().FromReader(layerReader)

if err := store.Layers.LinkStoreFileFrom(
layerDigest.Hex(), layerPath); err != nil && !os.IsExist(err) {

return fmt.Errorf("commit layer to store: %s", err)
}

layers = append(layers, image.Descriptor{
MediaType: image.MediaTypeLayer,
Size: layerInfo.Size(),
Digest: layerDigest,
})
}

// Import extracted dir content into image store -- manifest.json.
distManifest := image.DistributionManifest{
SchemaVersion: 2,
MediaType: image.MediaTypeManifest,
Config: image.Descriptor{
MediaType: image.MediaTypeConfig,
Size: configInfo.Size(),
Digest: configDigest,
},
Layers: layers,
}
store.SaveManifest(distManifest, imageName)

for _, replica := range replicas {
parsed := image.MustParseName(replica)
store.SaveManifest(distManifest, parsed)
}
}

return nil
}
1 change: 1 addition & 0 deletions bin/makisu/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ func Execute() {
rootCmd.AddCommand(getBuildCmd().Command)
rootCmd.AddCommand(getVersionCmd())
rootCmd.AddCommand(getPullCmd().Command)
rootCmd.AddCommand(getPushCmd().Command)
if err := rootCmd.Execute(); err != nil {
log.Error(err)
os.Exit(1)
Expand Down
11 changes: 6 additions & 5 deletions bin/makisu/cmd/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package cmd

import (
ctx "context"
"errors"
"fmt"
"io/ioutil"
"net/http"
Expand All @@ -37,12 +38,12 @@ import (
"github.com/uber/makisu/lib/utils/stringset"
)

func (cmd *buildCmd) initRegistryConfig() error {
if cmd.registryConfig == "" {
func initRegistryConfig(registryConfig string) error {
if registryConfig == "" {
return nil
}
cmd.registryConfig = os.ExpandEnv(cmd.registryConfig)
if err := registry.UpdateGlobalConfig(cmd.registryConfig); err != nil {
registryConfig = os.ExpandEnv(registryConfig)
if err := registry.UpdateGlobalConfig(registryConfig); err != nil {
return fmt.Errorf("init registry config: %s", err)
}
return nil
Expand Down Expand Up @@ -89,7 +90,7 @@ func (cmd *buildCmd) getDockerfile(contextDir string) ([]*dockerfile.Stage, erro
func (cmd *buildCmd) getTargetImageName() (image.Name, error) {
if cmd.tag == "" {
msg := "please specify a target image name: makisu build -t=(<registry:port>/)<repo>:<tag> ./"
return image.Name{}, fmt.Errorf(msg)
return image.Name{}, errors.New(msg)
}

// Parse the target's image name into its components.
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
Expand Down
2 changes: 1 addition & 1 deletion lib/builder/build_stage.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ func (stage *buildStage) saveManifest(
}

manifestPath := manifestFile.Name()
// Remove temp file after hard-linked to manifest store
// Remove temp file after hard-linked to manifest store.
defer os.Remove(manifestPath)

if err := ioutil.WriteFile(manifestPath, manifestJSON, 0755); err != nil {
Expand Down
18 changes: 6 additions & 12 deletions lib/docker/cli/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,7 @@ import (
"github.com/uber/makisu/lib/stream"
)

// ImageTarer contains a Tar function that returns a reader to the resulting
// tar file.
type ImageTarer interface {
Tar(registry, repo, tag string) (io.Reader, error)
}

// DefaultImageTarer is the default implementation of the ImageTarer interface.
// DefaultImageTarer exports/imports images from an ImageStore.
type DefaultImageTarer struct {
store *storage.ImageStore
}
Expand All @@ -48,8 +42,8 @@ func NewDefaultImageTarer(store *storage.ImageStore) DefaultImageTarer {
}
}

// CreateTarReadCloser creates a new tar from the inputs and returns a reader
// that automatically closes on EOF.
// CreateTarReadCloser exports an image from the image store as a tar, and
// returns a reader for the tar that automatically closes on EOF.
func (tarer DefaultImageTarer) CreateTarReadCloser(imageName image.Name) (io.Reader, error) {
dir, err := tarer.createTarDir(imageName)
if err != nil {
Expand All @@ -75,8 +69,8 @@ func (tarer DefaultImageTarer) CreateTarReadCloser(imageName image.Name) (io.Rea
return reader, nil
}

// CreateTarReader creates a new tar from the inputs and returns a simple reader
// to that file.
// CreateTarReader exports an image from the image store as a tar, and returns a
// reader for the tar.
func (tarer DefaultImageTarer) CreateTarReader(imageName image.Name) (io.Reader, error) {
dir, err := tarer.createTarDir(imageName)
if err != nil {
Expand Down Expand Up @@ -104,8 +98,8 @@ func (tarer DefaultImageTarer) createTarDir(imageName image.Name) (string, error
return "", err
}

repo, tag := imageName.GetRepository(), imageName.GetTag()
// Create tmp file for target tar.
repo, tag := imageName.GetRepository(), imageName.GetTag()
dir := filepath.Join(tarer.store.SandboxDir, repo, tag)
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
Expand Down
Loading

0 comments on commit 9ed3b61

Please sign in to comment.