From fd0574d5e0a65f17095fea3201d8f86a00233608 Mon Sep 17 00:00:00 2001 From: alex-codefresh <33512662+alex-codefresh@users.noreply.github.com> Date: Wed, 9 Dec 2020 11:28:27 +0300 Subject: [PATCH] Cr 2066 (#14) * Refactor Dockerfile * Bump version * Fix bolter args Co-authored-by: Pavel Nosovets --- Dockerfile | 28 +- cleaner/dind-cleaner/cmd/main.go | 297 ++++++++++++++++++ cleaner/dind-cleaner/glide.lock | 65 ++++ cleaner/dind-cleaner/glide.yaml | 8 + cleaner/dind-cleaner/test/Dockerfile | 12 + .../dind-cleaner/test/dind-config-no-tls.json | 5 + .../test/res/preserved-images-list | 8 + cleaner/dind-cleaner/test/run-dind.sh | 36 +++ run.sh | 2 +- service.yaml | 2 +- 10 files changed, 456 insertions(+), 7 deletions(-) create mode 100644 cleaner/dind-cleaner/cmd/main.go create mode 100644 cleaner/dind-cleaner/glide.lock create mode 100644 cleaner/dind-cleaner/glide.yaml create mode 100644 cleaner/dind-cleaner/test/Dockerfile create mode 100644 cleaner/dind-cleaner/test/dind-config-no-tls.json create mode 100644 cleaner/dind-cleaner/test/res/preserved-images-list create mode 100755 cleaner/dind-cleaner/test/run-dind.sh diff --git a/Dockerfile b/Dockerfile index 109344d..7952e77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,30 @@ ARG DOCKER_VERSION=18.09.5 -FROM quay.io/prometheus/node-exporter:v0.15.1 AS node-exporter -# install node-exporter +# dind-cleaner +FROM golang:1.9.2 AS cleaner +RUN curl https://glide.sh/get | sh -FROM codefresh/dind-cleaner:v1.1 AS dind-cleaner +COPY cleaner/dind-cleaner/glide* /go/src/github.com/codefresh-io/dind-cleaner/ +WORKDIR /go/src/github.com/codefresh-io/dind-cleaner/ -FROM codefresh/bolter AS bolter +RUN mkdir -p /go/src/github.com/codefresh-io/dind-cleaner/{cmd,pkg} +RUN glide install --strip-vendor && rm -rf /root/.glide +COPY cleaner/dind-cleaner/cmd ./cmd/ + +RUN CGO_ENABLED=0 go build -o /usr/local/bin/dind-cleaner ./cmd && \ + chmod +x /usr/local/bin/dind-cleaner && \ + rm -rf /go/* + +# bolter +FROM golang:1.12.6-alpine3.9 AS bolter +RUN apk add git +RUN go get -u github.com/hasit/bolter + +# node-exporter +FROM quay.io/prometheus/node-exporter:v1.0.0 AS node-exporter + +# Main FROM docker:${DOCKER_VERSION}-dind RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.11/main' >> /etc/apk/repositories \ @@ -15,7 +33,7 @@ RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.11/main' >> /etc/apk/repositor && rm -rf /var/cache/apk/* COPY --from=node-exporter /bin/node_exporter /bin/ -COPY --from=dind-cleaner /usr/local/bin/dind-cleaner /bin/ +COPY --from=cleaner /usr/local/bin/dind-cleaner /bin/ COPY --from=bolter /go/bin/bolter /bin/ WORKDIR /dind diff --git a/cleaner/dind-cleaner/cmd/main.go b/cleaner/dind-cleaner/cmd/main.go new file mode 100644 index 0000000..a74dfe0 --- /dev/null +++ b/cleaner/dind-cleaner/cmd/main.go @@ -0,0 +1,297 @@ +/* +Copyright 2018 The Codefresh Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "time" + "flag" + "os" + "bufio" + "github.com/golang/glog" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "golang.org/x/net/context" +) + +func readFileLines(path string) ([]string, error) { + var lines []string + if path == "" { + return lines, nil + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, scanner.Err() +} + +var dryRun *bool +const ( + cmdImages = "images" + + statusFound = "found" + statusRemoved = "removed" + statusRetainedByList = "retainedByList" + statusRetainedByDate = "retainedByDate" + statusChildRetained = "childRetained" + statusChildFailedToRemove = "childFailedToRemove" + statusFailedToRemove = "failedToRemove" +) + +func _stringInList(list []string, s string) bool { + for _, a := range list { + if a == s { + return true + } + } + return false +} + +func cleanImages(retainedImagesList []string, retainPeriod int64) { + glog.Infof("Entering cleanImages, length of retainedImagesList = %d", len(retainedImagesList)) + if os.Getenv("DOCKER_API_VERSION") == "" { + os.Setenv("DOCKER_API_VERSION", "1.35") + } + + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + + type imageToCleanStruct = struct { + ID string + Created int64 + ParentID string + status string + tags []string + childrenIDs map[string]string + size int64 + } + + + /* + Purpose: remove images starting from first child excluding ids in retainedImagesList + Logic: + 1. get All images (with All=true) + 2. fill map of imageToCleanStruct - for each image fill its children in the map of [id]"status" + 3. find images with no children + 4. loop by found images with no children and delete them, then update childrenList of whole map of imageToCleanStruct. + Skip deletion for images in retainedImagesList + --- Repeat 3-4 until images to delete found + + */ + + // 1. Get All Images + ctx := context.Background() + imagesFullList, err := cli.ImageList(ctx, types.ImageListOptions{All: true}) + if err != nil { + panic(err) + } + + glog.Infof("Found %d images in docker", len(imagesFullList)) + + currentTs := time.Now().Unix() + // 2. fill map of imageToCleanStruct + images := make(map[string]*imageToCleanStruct) + for _, img := range imagesFullList { + images[img.ID] = &imageToCleanStruct{ + ID: img.ID, + Created: img.Created, + ParentID: img.ParentID, + status: statusFound, + tags: img.RepoTags, + size: img.Size, + childrenIDs: make(map[string]string), + } + } + + glog.Infof("Calculating child images ...") + for imageID, img := range images { + if img.ParentID != "" { + parentImage, parentImageInList := images[img.ParentID] + if parentImageInList { + parentImage.childrenIDs[imageID] = statusFound + } + } + } + + // Loop until found some imagesToDelete + var imagesToDelete []string + loopCount := 0 + for { + imagesToDelete = nil + loopCount++ + // 3. finding all images with no children + glog.Infof("\n\n#################\n------ Loop %d - finding images without any children to remove ...", loopCount) + for imageID, img := range images { + if len(img.childrenIDs) == 0 && (img.status == statusFound || img.status == statusChildFailedToRemove) { + imagesToDelete = append(imagesToDelete, imageID) + } + } + + if len(imagesToDelete) == 0 { + glog.Infof("Stopping - no images leave to remove ...") + break + } + + // 4. Loop by found images and delete|retain , then update whole images map + glog.Infof("Found %d images with no children", len(imagesToDelete)) + for _, imageID := range imagesToDelete { + glog.Infof("\n Check if to remove image %s - %v", imageID, images[imageID].tags) + // checking if image in retained list + + if _stringInList(retainedImagesList, imageID) { + glog.Infof(" Skiping image %s - %v , it appears in retained list", imageID, images[imageID].tags) + images[imageID].status = statusRetainedByList + } else if retainPeriod > 0 && images[imageID].Created > 0 && images[imageID].Created < currentTs && + currentTs - images[imageID].Created < retainPeriod { + + glog.Infof(" Skiping image %s - %v , its created more than retainPeriod %d seconds ago", imageID, images[imageID].tags, retainPeriod) + images[imageID].status = statusRetainedByDate + } else { + glog.Infof(" Deleting image %s - %v", imageID, images[imageID].tags) + // add image delete here + var err error + if !*dryRun { + _, err = cli.ImageRemove(ctx, imageID, types.ImageRemoveOptions{Force: true, PruneChildren: false}) + } else { + glog.Infof( "DRY RUN - do not actually delete") + } + + if err == nil { + glog.Infof(" image %s - %v has been deleted", imageID, images[imageID].tags) + images[imageID].status = statusRemoved + } else { + glog.Infof(" FAILED to delete image %s - %v - %v", imageID, images[imageID].tags, err) + images[imageID].status = statusFailedToRemove + } + } + + glog.Infof(" setting image status to %s", images[imageID].status) + for _, img := range images { + if _, ok := img.childrenIDs[imageID]; ok { + if images[imageID].status == statusRemoved { + glog.Infof(" deleting the child from parent image %s - %v", img.ID, img.tags) + delete(img.childrenIDs, imageID) + } else if images[imageID].status == statusRetainedByList || images[imageID].status == statusRetainedByDate { + glog.Infof(" setting child status %s for image %s - %v", images[imageID].status, img.ID, img.tags) + img.childrenIDs[imageID] = images[imageID].status + img.status = statusChildRetained + + } else if images[imageID].status == statusFailedToRemove { + glog.Infof(" setting child status %s and deleting the from parent image %s - %v", images[imageID].status, img.ID, img.tags) + delete(img.childrenIDs, imageID) + img.status = statusChildFailedToRemove + } + } + } + } + } + + glog.Info("\n################\nPrinting results ..") + var totalImagesSize, removedSize, retainedByListSize, retainedByDateSize, failedToRemoveSize int64 + for _, img := range images { + glog.Infof("%s: %v - %s, size = %d", img.status, img.tags, img.ID, img.size) + for childID, childStatus := range img.childrenIDs { + glog.Infof(" Child: %s - %s (grandchild retained)", childID, childStatus) + } + + totalImagesSize += img.size + switch img.status { + case statusRemoved: + removedSize += img.size + case statusRetainedByList: + retainedByListSize += img.size + case statusRetainedByDate: + retainedByDateSize += img.size + case statusFailedToRemove: + failedToRemoveSize += img.size + } + } + + glog.Infof("\n-----------\n" + + " total images shared size: %.3f Mb \n" + + " removed shared size: %.3f Mb \n" + + "retained shared by list size: %.3f Mb \n" + + "retained shared by date size: %.3f Mb \n" + + " failed to remove size: %.3f Mb ", + float64(totalImagesSize)/1024/1024.0, + float64(removedSize)/1024/1024.0, + float64(retainedByListSize)/1024/1024.0, + float64(retainedByDateSize)/1024/1024.0, + float64(failedToRemoveSize)/1024/1024.0) +} + +func main() { + + usage := ` +Usage: dind-cleaner [options] + +Commands: + images [--retained-images-file] [--dry-run] +` + flag.Parse() + flag.Set("v", "4") + flag.Set("alsologtostderr", "true") + validCommands := []string{"images"} + if len(os.Args) < 2 { + glog.Errorf("%s", usage) + os.Exit(2) + } else if !_stringInList(validCommands,os.Args[1]) { + glog.Errorf("Invalid command %s\n%s", os.Args[1], usage) + os.Exit(2) + } + + imagesCommand := flag.NewFlagSet("images", flag.ExitOnError) + retainedImagesListFile := imagesCommand.String("retained-images-file", "", "Retained images list file") + imageRetainPeriod := imagesCommand.Int64("image-retain-period", 86400, "image retain period") + + dryRun = imagesCommand.Bool("dry-run", false, "dry run - only print actions") + + switch os.Args[1] { + case "images": + imagesCommand.Parse(os.Args[2:]) + imagesCommand.Set("v", "4") + default: + glog.Errorf("%q is not valid command.\n", os.Args[1]) + os.Exit(2) + } + + if os.Getenv("CLEANER_DRY_RUN") != "" { + *dryRun = true + } + + + glog.Infof("\n----------------\n Started dind-cleaner") + + glog.Infof("First verson - only image cleaner. " + + "retainedImagesListFile = %s " + + "retainedImagesPeriod = %d " + + "dry-run = %t" , *retainedImagesListFile, *imageRetainPeriod, *dryRun) + + retainedImagesList, err := readFileLines(*retainedImagesListFile) + if err != nil { + glog.Errorf("Failed to read file %s: %v", *retainedImagesListFile, err) + } + cleanImages(retainedImagesList, *imageRetainPeriod) +} diff --git a/cleaner/dind-cleaner/glide.lock b/cleaner/dind-cleaner/glide.lock new file mode 100644 index 0000000..cd10777 --- /dev/null +++ b/cleaner/dind-cleaner/glide.lock @@ -0,0 +1,65 @@ +hash: 5a99cb1e130cfafa2a233a9256d2dece615aefd763d10334f2c7d1f0654f40c1 +updated: 2018-08-26T20:10:59.045652849+03:00 +imports: +- name: github.com/docker/distribution + version: edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c + subpackages: + - digestset + - reference +- name: github.com/docker/docker + version: fe3bc75cc44ec90afbd126e4576f9f4f829fc0bc + subpackages: + - api + - api/types + - api/types/blkiodev + - api/types/container + - api/types/events + - api/types/filters + - api/types/image + - api/types/mount + - api/types/network + - api/types/registry + - api/types/strslice + - api/types/swarm + - api/types/swarm/runtime + - api/types/time + - api/types/versions + - api/types/volume + - client + - errdefs +- name: github.com/docker/go-connections + version: 97c2040d34dfae1d1b1275fa3a78dbdd2f41cf7e + subpackages: + - nat + - sockets + - tlsconfig +- name: github.com/docker/go-units + version: 47565b4f722fb6ceae66b95f853feed578a4a51c +- name: github.com/gogo/protobuf + version: c0656edd0d9eab7c66d1eb0c568f9039345796f7 + subpackages: + - proto +- name: github.com/golang/glog + version: 44145f04b68cf362d9c4df2182967c2275eaefed +- name: github.com/Microsoft/go-winio + version: 97e4973ce50b2ff5f09635a57e2b88a037aae829 +- name: github.com/opencontainers/go-digest + version: a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb +- name: github.com/opencontainers/image-spec + version: e562b04403929d582d449ae5386ff79dd7961a11 + subpackages: + - specs-go + - specs-go/v1 +- name: github.com/pkg/errors + version: 816c9085562cd7ee03e7f8188a1cfd942858cded +- name: golang.org/x/net + version: 1c05540f6879653db88113bc4a2b70aec4bd491f + subpackages: + - context + - context/ctxhttp + - proxy +- name: golang.org/x/sys + version: a2e06a18b0d52d8cb2010e04b372a1965d8e3439 + subpackages: + - windows +testImports: [] diff --git a/cleaner/dind-cleaner/glide.yaml b/cleaner/dind-cleaner/glide.yaml new file mode 100644 index 0000000..b5ac7a9 --- /dev/null +++ b/cleaner/dind-cleaner/glide.yaml @@ -0,0 +1,8 @@ +package: github.com/codefresh-io/dind-cleaner +import: +- package: github.com/docker/docker + version: ~18.06 + subpackages: + - api/types + - client +- package: github.com/golang/glog diff --git a/cleaner/dind-cleaner/test/Dockerfile b/cleaner/dind-cleaner/test/Dockerfile new file mode 100644 index 0000000..368980d --- /dev/null +++ b/cleaner/dind-cleaner/test/Dockerfile @@ -0,0 +1,12 @@ +FROM alpine AS base +RUN echo ddddd +ARG TEST_ARG +RUN echo hi + +FROM base AS dependencies +RUN echo hi stage 2 + + +FROM base +COPY --from=dependencies /tmp /tmp2 +RUN echo test \ No newline at end of file diff --git a/cleaner/dind-cleaner/test/dind-config-no-tls.json b/cleaner/dind-cleaner/test/dind-config-no-tls.json new file mode 100644 index 0000000..2432baa --- /dev/null +++ b/cleaner/dind-cleaner/test/dind-config-no-tls.json @@ -0,0 +1,5 @@ +{ + "hosts": [ "unix:///var/run/docker.sock", + "tcp://0.0.0.0:1300"], + "storage-driver": "overlay2" +} \ No newline at end of file diff --git a/cleaner/dind-cleaner/test/res/preserved-images-list b/cleaner/dind-cleaner/test/res/preserved-images-list new file mode 100644 index 0000000..91c5806 --- /dev/null +++ b/cleaner/dind-cleaner/test/res/preserved-images-list @@ -0,0 +1,8 @@ + +alpine:latest +sha256:267af47eff8039135b62b3570ceab02e9f831cdd1005db4e50b50998d8e2dfc2 +sha256:9ec95afac19346da3a701bec93145ac3ca6f70f796e193526baa56b5d861f35d +sha256:a280fe96a32d2080ebc48ec774f3090692dbc278cf4f7571573545bc8311fbd2 +sha256:c56db19e8b5c6e53a131d2fbb8a03373a66a19636ed9d8f5afc66f0242d8fda8 +sha256:caf27325b298a6730837023a8a342699c8b7b388b8d878966b064a1320043019 +sha256:e9bcfacdda70043b3c9e9e2d1a5d4548bfaeaa242a391549c15d0e298899227d diff --git a/cleaner/dind-cleaner/test/run-dind.sh b/cleaner/dind-cleaner/test/run-dind.sh new file mode 100755 index 0000000..b31fbfb --- /dev/null +++ b/cleaner/dind-cleaner/test/run-dind.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +DIR=$(dirname $0) + +CONTAINER_NAME=${CONTAINER_NAME:-dind-cleaner-test} +DIND_PORT=${DIND_PORT:-1300} +DIND_IMAGE=codefresh/dind:18.09-v16 +#DIND_IMAGE=docker:18.09.2-dind +#DIND_IMAGE_COMMAND=dockerd + + +CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' ${CONTAINER_NAME} 2>/dev/null) +if [[ $? == 0 ]]; then + echo "Container ${CONTAINER_NAME} is already ${CONTAINER_STATUS} +" + read -r -p "Do you want to recreate it? [y/N] " response + if [[ "$response" =~ ^([yY][eE][sS]|[yY])+$ ]] + then + echo "Removing container ${CONTAINER_NAME} ..." + docker rm -fv ${CONTAINER_NAME} + else + echo "Exiting..." + exit 0 + fi +fi + +docker run -d --privileged -p ${DIND_PORT}:1300 --name $CONTAINER_NAME \ + -v dind-cleaner-test:/var/lib/docker \ + -v $(realpath $DIR/dind-config-no-tls.json):/etc/docker/daemon.json \ + $DIND_IMAGE $DIND_IMAGE_COMMAND + +export DOCKER_HOST=localhost:1300 + +docker info +echo "export DOCKER_HOST=${DOCKER_HOST}" +bash -ca "export DOCKER_HOST=${DOCKER_HOST}" \ No newline at end of file diff --git a/run.sh b/run.sh index 9342f40..3a4f674 100755 --- a/run.sh +++ b/run.sh @@ -159,7 +159,7 @@ do if [[ -f ${CONTEINERD_DB} ]]; then echo "Checking if another dockerd is running on same ${DOCKERD_DATA_ROOT} boltdb $CONTEINERD_DB is locked" CNT=0 - while ! bolter -f ${CONTEINERD_DB} + while ! bolter --file ${CONTEINERD_DB} do [[ -n "${SIGTERM}" ]] && break 2 echo "$(date) - Waiting for containerd boltd ${CONTEINERD_DB}" diff --git a/service.yaml b/service.yaml index ebee4d2..4a3ca30 100644 --- a/service.yaml +++ b/service.yaml @@ -1 +1 @@ -version: 1.24.3 \ No newline at end of file +version: 1.24.4