Add foundations for FIPS flavor (#31004)

Co-authored-by: jeremy-hanna <[email protected]>

.gitlab-ci.yml

@@ -173,49 +173,49 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v48372186-ff395e52
+  DATADOG_AGENT_BUILDIMAGES: v48815877-9bfad02c
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v48372186-ff395e52
+  DATADOG_AGENT_WINBUILDIMAGES: v48815877-9bfad02c
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v48372186-ff395e52
+  DATADOG_AGENT_ARMBUILDIMAGES: v48815877-9bfad02c
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v48372186-ff395e52
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v48815877-9bfad02c
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v48372186-ff395e52
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v48815877-9bfad02c
   # New images to enable different version per image - not used yet
-  CI_IMAGE_BTF_GEN: v48372186-ff395e52
+  CI_IMAGE_BTF_GEN: v48815877-9bfad02c
   CI_IMAGE_BTF_GEN_SUFFIX: ""
-  CI_IMAGE_DEB_X64: v48372186-ff395e52
+  CI_IMAGE_DEB_X64: v48815877-9bfad02c
   CI_IMAGE_DEB_X64_SUFFIX: ""
-  CI_IMAGE_DEB_ARM64: v48372186-ff395e52
+  CI_IMAGE_DEB_ARM64: v48815877-9bfad02c
   CI_IMAGE_DEB_ARM64_SUFFIX: ""
-  CI_IMAGE_DEB_ARMHF: v48372186-ff395e52
+  CI_IMAGE_DEB_ARMHF: v48815877-9bfad02c
   CI_IMAGE_DEB_ARMHF_SUFFIX: ""
-  CI_IMAGE_DD_AGENT_TESTING: v48372186-ff395e52
+  CI_IMAGE_DD_AGENT_TESTING: v48815877-9bfad02c
   CI_IMAGE_DD_AGENT_TESTING_SUFFIX: ""
-  CI_IMAGE_DOCKER_X64: v48372186-ff395e52
+  CI_IMAGE_DOCKER_X64: v48815877-9bfad02c
   CI_IMAGE_DOCKER_X64_SUFFIX: ""
-  CI_IMAGE_DOCKER_ARM64: v48372186-ff395e52
+  CI_IMAGE_DOCKER_ARM64: v48815877-9bfad02c
   CI_IMAGE_DOCKER_ARM64_SUFFIX: ""
-  CI_IMAGE_GITLAB_AGENT_DEPLOY: v48372186-ff395e52
+  CI_IMAGE_GITLAB_AGENT_DEPLOY: v48815877-9bfad02c
   CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX: ""
-  CI_IMAGE_LINUX_GLIBC_2_17_X64: v48372186-ff395e52
+  CI_IMAGE_LINUX_GLIBC_2_17_X64: v48815877-9bfad02c
   CI_IMAGE_LINUX_GLIBC_2_17_X64_SUFFIX: ""
-  CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v48372186-ff395e52
+  CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v48815877-9bfad02c
   CI_IMAGE_LINUX_GLIBC_2_23_ARM64_SUFFIX: ""
-  CI_IMAGE_SYSTEM_PROBE_X64: v48372186-ff395e52
+  CI_IMAGE_SYSTEM_PROBE_X64: v48815877-9bfad02c
   CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: ""
-  CI_IMAGE_SYSTEM_PROBE_ARM64: v48372186-ff395e52
+  CI_IMAGE_SYSTEM_PROBE_ARM64: v48815877-9bfad02c
   CI_IMAGE_SYSTEM_PROBE_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_X64: v48372186-ff395e52
+  CI_IMAGE_RPM_X64: v48815877-9bfad02c
   CI_IMAGE_RPM_X64_SUFFIX: ""
-  CI_IMAGE_RPM_ARM64: v48372186-ff395e52
+  CI_IMAGE_RPM_ARM64: v48815877-9bfad02c
   CI_IMAGE_RPM_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_ARMHF: v48372186-ff395e52
+  CI_IMAGE_RPM_ARMHF: v48815877-9bfad02c
   CI_IMAGE_RPM_ARMHF_SUFFIX: ""
-  CI_IMAGE_WIN_1809_X64: v48372186-ff395e52
+  CI_IMAGE_WIN_1809_X64: v48815877-9bfad02c
   CI_IMAGE_WIN_1809_X64_SUFFIX: ""
-  CI_IMAGE_WIN_LTSC2022_X64: v48372186-ff395e52
+  CI_IMAGE_WIN_LTSC2022_X64: v48815877-9bfad02c
   CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: ""
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded

.gitlab/container_build/docker_linux.yml

@@ -89,6 +89,33 @@ docker_build_agent7_arm64:
     TAG_SUFFIX: -7
     BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-arm64.tar.xz
 
+# build agent7 fips image
+docker_build_fips_agent7:
+  extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs:
+    - job: datadog-agent-7-x64-fips
+  variables:
+    IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
+    BUILD_CONTEXT: Dockerfiles/agent
+    TAG_SUFFIX: -7-fips
+    BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz
+
+docker_build_fips_agent7_arm64:
+  extends: [.docker_build_job_definition_arm64, .docker_build_artifact]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs:
+    - job: datadog-agent-7-arm64-fips
+  variables:
+    IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
+    BUILD_CONTEXT: Dockerfiles/agent
+    TAG_SUFFIX: -7-fips
+    BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz
+
 # build agent7 jmx image
 docker_build_agent7_jmx:
   extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
@@ -116,6 +143,32 @@ docker_build_agent7_jmx_arm64:
     TAG_SUFFIX: -7-jmx
     BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-arm64.tar.xz
 
+docker_build_fips_agent7_jmx:
+  extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs:
+    - job: datadog-agent-7-x64-fips
+  variables:
+    IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
+    BUILD_CONTEXT: Dockerfiles/agent
+    TAG_SUFFIX: -7-fips-jmx
+    BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-amd64.tar.xz
+
+docker_build_fips_agent7_arm64_jmx:
+  extends: [.docker_build_job_definition_arm64, .docker_build_artifact]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs:
+    - job: datadog-agent-7-arm64-fips
+  variables:
+    IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
+    BUILD_CONTEXT: Dockerfiles/agent
+    TAG_SUFFIX: -7-fips-jmx
+    BUILD_ARG: --target test --build-arg DD_AGENT_ARTIFACT=datadog-fips-agent-7*-arm64.tar.xz
+
 # build agent7 UA image
 docker_build_ot_agent7:
   extends: [.docker_build_job_definition_amd64, .docker_build_artifact]

.gitlab/deploy_packages/nix.yml

@@ -14,6 +14,18 @@ deploy_packages_deb-arm64-7:
   variables:
     PACKAGE_ARCH: arm64
 
+deploy_packages_deb-x64-7-fips:
+  extends: .deploy_packages_deb-7
+  needs: [ agent_deb-x64-a7-fips ]
+  variables:
+    PACKAGE_ARCH: amd64
+
+deploy_packages_deb-arm64-7-fips:
+  extends: .deploy_packages_deb-7
+  needs: [ agent_deb-arm64-a7-fips ]
+  variables:
+    PACKAGE_ARCH: arm64
+
 deploy_packages_heroku_deb-x64-7:
   extends: .deploy_packages_deb-7
   needs: [ agent_heroku_deb-x64-a7 ]
@@ -62,6 +74,18 @@ deploy_packages_rpm-arm64-7:
   variables:
     PACKAGE_ARCH: aarch64
 
+deploy_packages_rpm-x64-7-fips:
+  extends: .deploy_packages_rpm-7
+  needs: [ agent_rpm-x64-a7-fips ]
+  variables:
+    PACKAGE_ARCH: x86_64
+
+deploy_packages_rpm-arm64-7-fips:
+  extends: .deploy_packages_rpm-7
+  needs: [ agent_rpm-arm64-a7-fips ]
+  variables:
+    PACKAGE_ARCH: aarch64
+
 deploy_packages_iot_rpm-x64-7:
   extends: .deploy_packages_rpm-7
   needs: [ iot_agent_rpm-x64 ]
@@ -98,6 +122,18 @@ deploy_packages_suse_rpm-arm64-7:
   variables:
     PACKAGE_ARCH: aarch64
 
+deploy_packages_suse_rpm-x64-7-fips:
+  extends: .deploy_packages_suse_rpm-7
+  needs: [ agent_suse-x64-a7-fips ]
+  variables:
+    PACKAGE_ARCH: x86_64
+
+deploy_packages_suse_rpm-arm64-7-fips:
+  extends: .deploy_packages_suse_rpm-7
+  needs: [ agent_suse-arm64-a7-fips ]
+  variables:
+    PACKAGE_ARCH: aarch64
+
 deploy_packages_iot_suse_rpm-x64-7:
   extends: .deploy_packages_suse_rpm-7
   needs: [ iot_agent_suse-x64 ]

.gitlab/dev_container_deploy/docker_linux.yml

@@ -32,6 +32,24 @@ dev_branch_multiarch-a7:
       - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64
         IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx
 
+dev_branch_multiarch-fips:
+  extends: .docker_publish_job_definition
+  stage: dev_container_deploy
+  rules: !reference [.manual]
+  needs:
+    - docker_build_fips_agent7
+    - docker_build_fips_agent7_arm64
+    - docker_build_fips_agent7_jmx
+    - docker_build_fips_agent7_arm64_jmx
+  variables:
+    IMG_REGISTRIES: dev
+  parallel:
+    matrix:
+      - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-arm64
+        IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-fips
+      - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-arm64
+        IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-fips-jmx
+
 dev_branch_multiarch-dogstatsd:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy

.gitlab/internal_image_deploy/internal_image_deploy.yml

@@ -47,6 +47,51 @@ docker_trigger_internal:
       --variable TARGET_ENV
       --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS"
 
+docker_trigger_internal-fips:
+  stage: internal_image_deploy
+  rules: !reference [.on_deploy_internal_or_manual]
+  needs:
+    - job: docker_build_fips_agent7
+      artifacts: false
+    - job: docker_build_fips_agent7_arm64
+      artifacts: false
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  variables:
+    DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo
+    IMAGE_VERSION: tmpl-v11
+    IMAGE_NAME: datadog-agent
+    RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-fips
+    BUILD_TAG: ${CI_COMMIT_REF_SLUG}-fips
+    TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips
+    TMPL_SRC_REPO: ci/datadog-agent/agent
+    RELEASE_STAGING: "true"
+  script:
+    - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN
+    - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi
+    - |
+      if [ "$BUCKET_BRANCH" = "nightly" ]; then
+        RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"
+        TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly"
+      fi
+    - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi
+    - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600
+      --variable IMAGE_VERSION
+      --variable IMAGE_NAME
+      --variable RELEASE_TAG
+      --variable BUILD_TAG
+      --variable TMPL_SRC_IMAGE
+      --variable TMPL_SRC_REPO
+      --variable RELEASE_STAGING
+      --variable RELEASE_PROD
+      --variable DYNAMIC_BUILD_RENDER_RULES
+      --variable APPS
+      --variable BAZEL_TARGET
+      --variable DDR
+      --variable DDR_WORKFLOW_ID
+      --variable TARGET_ENV
+      --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS"
+
 docker_trigger_internal-ot:
   stage: internal_image_deploy
   rules: !reference [.on_deploy_internal_or_manual]

.gitlab/package_build/linux.yml

@@ -67,6 +67,12 @@
   before_script:
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
+.agent_fips_build:
+  variables:
+    FLAVOR: fips
+  before_script:
+    - export RELEASE_VERSION=$RELEASE_VERSION_7
+
 # build Agent 7 binaries for x86_64
 datadog-agent-7-x64:
   extends: [.agent_build_common, .agent_build_x86, .agent_7_build]
@@ -83,6 +89,14 @@ datadog-ot-agent-7-x64:
 datadog-ot-agent-7-arm64:
   extends: [.agent_build_common, .agent_build_arm64, .ot_agent_7_build]
 
+# build Agent 7 binaries for x86_64 with FIPS
+datadog-agent-7-x64-fips:
+  extends: [.agent_build_common, .agent_build_x86, .agent_fips_build]
+
+# build Agent 7 binaries for arm64 with FIPS
+datadog-agent-7-arm64-fips:
+  extends: [.agent_build_common, .agent_build_arm64, .agent_fips_build]
+
 .iot-agent-common:
   extends: .agent_build_common
   needs: ["go_mod_tidy_check", "go_deps"]

.gitlab/packaging/deb.yml

@@ -56,6 +56,26 @@ agent_deb-arm64-a7:
   variables:
     DD_PROJECT: "agent"
 
+agent_deb-x64-a7-fips:
+  extends: [.package_deb_common, .package_deb_x86, .package_deb_agent_7]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs: ["datadog-agent-7-x64-fips"]
+  variables:
+    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    DD_PROJECT: "agent"
+
+agent_deb-arm64-a7-fips:
+  extends: [.package_deb_common, .package_deb_arm64, .package_deb_agent_7]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs: ["datadog-agent-7-arm64-fips"]
+  variables:
+    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    DD_PROJECT: "agent"
+
 .package_ot_deb_common:
   extends: [.package_deb_common]
   script:

.gitlab/packaging/rpm.yml

@@ -92,6 +92,35 @@ installer_rpm-amd64:
     # explicitly disable the check
     PACKAGE_REQUIRED_FILES_LIST: ""
 
+agent_rpm-x64-a7-fips:
+  extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_x86]
+  tags: ["arch:amd64"]
+  needs: ["datadog-agent-7-x64-fips"]
+  variables:
+    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    DD_PROJECT: agent
+
+agent_rpm-arm64-a7-fips:
+  extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_arm64]
+  needs: ["datadog-agent-7-arm64-fips"]
+  variables:
+    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    DD_PROJECT: agent
+
+agent_suse-x64-a7-fips:
+  extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_x86]
+  needs: ["datadog-agent-7-x64-fips"]
+  variables:
+    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    DD_PROJECT: agent
+
+agent_suse-arm64-a7-fips:
+  extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_arm64]
+  needs: ["datadog-agent-7-arm64-fips"]
+  variables:
+    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    DD_PROJECT: agent
+
 installer_rpm-arm64:
   extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_arm64]
   needs: ["installer-arm64"]

Dockerfiles/agent/Dockerfile

@@ -180,6 +180,13 @@ RUN tar xzf s6.tgz -C / --exclude="./bin" \
 # * https://datadoghq.atlassian.net/wiki/spaces/TS/pages/2615709591/Why+the+containerized+Agent+runs+as+root#Agent-user
 RUN [ "$(getent passwd dd-agent | cut -d: -f 3)" -eq 100 ]
 
+# Enable FIPS if needed
+RUN if [ -x /opt/datadog-agent/embedded/bin/fipsinstall.sh ]; then \
+  /opt/datadog-agent/embedded/bin/fipsinstall.sh; \
+fi
+# This is used by MSGO to enable FIPS mode so it won't affect the non-FIPS image
+ENV GOFIPS=1
+
 # Override the exit script by ours to fix --pid=host operations
 RUN  mv /etc/s6/init/init-stage3 /etc/s6/init/init-stage3-original
 COPY init-stage3          /etc/s6/init/init-stage3