Skip to content

Commit

Permalink
refactor updater with some improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@gdams
gdams committed Oct 25, 2023
1 parent 5916415 commit a4b2525
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 165 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ official-eclipse-temurin
*hotspot.txt
library/
.vscode/
__pycache__/
__pycache__/
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Containers
[![Docker Stars](https://img.shields.io/docker/stars/_/eclipse-temurin?style=flat-square)](https://hub.docker.com/r/_/eclipse-temurin)
[![DockerPulls](https://img.shields.io/docker/pulls/_/eclipse-temurin?label=Docker%20Pulls)](https://hub.docker.com/_/eclipse-temurin)
[![Docker Stars](https://img.shields.io/docker/stars/_/eclipse-temurin?label=Docker%20Stars)](https://hub.docker.com/r/_/eclipse-temurin)

This repository contains the Dockerfiles for the official [Adoptium](https://adoptium.net) images of the Eclipse Temurin distribution (OpenJDK). These images are made available in Docker Hub.

Expand Down Expand Up @@ -28,7 +28,7 @@ A [Updater GitHub Action](.github/workflows/updater.yml) runs every 30 mins whic

#### generate_dockerfiles.py

[`./update_all.sh`](./update_all.sh) is a wrapper script to control what is passed into [`./update_multiarch.sh`](./update_multiarch.sh).
[`./generate_dockerfiles.py`](./generate_dockerfiles.py) is a python script which uses the jinja templates defined in [docker_templates](./docker_templates/) to generate the docker updates. It uses the Adoptium API to fetch the lastest artefacts for each release.

### Manual Release

Expand Down
11 changes: 7 additions & 4 deletions config/hotspot.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@ supported_distributions:
Versions: [8, 11, 17, 21]

configurations:
linux:
- directory: ubuntu/jammy
image: ubuntu:22.04
architectures: [aarch64, armv7l, ppc64le, s390x, x64]
architectures: [aarch64, arm, ppc64le, s390x, x64]
os: ubuntu

- directory: ubuntu/focal
architectures: [aarch64, armv7l, ppc64le, s390x, x64]
architectures: [aarch64, arm, ppc64le, s390x, x64]
image: ubuntu:20.04
deprecated: 20
os: ubuntu
Expand All @@ -38,12 +39,14 @@ configurations:
architectures: [aarch64, ppc64le, s390x, x64]
image: redhat/ubi9-minimal
os: ubi9-minimal


alpine-linux:
- directory: alpine
architectures: [aarch64, x64]
image: alpine:3.18
os: alpine-linux


windows:
- directory: windows/windowsservercore-1809
architectures: [x64]
image: mcr.microsoft.com/windows/servercore:1809
Expand Down
1 change: 1 addition & 0 deletions dockerhub_doc_config_update.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ else
fi

supported_versions="8 11 17 21"
latest_version="21"
all_jvms="hotspot"
all_packages="jdk jre"

Expand Down
205 changes: 100 additions & 105 deletions generate_dockerfiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"User-Agent": "Adoptium Dockerfile Updater",
}


def archHelper(arch, os_family):
if arch == "aarch64":
return "aarch64|arm64"
Expand All @@ -42,118 +43,112 @@ def archHelper(arch, os_family):
return arch


def osFamilyHelper(os):
if os == "ubuntu":
return "linux"
elif os == "centos":
return "linux"
elif os == "ubi9-minimal":
return "linux"
elif os == "nanoserver":
return "windows"
elif os == "servercore":
return "windows"
else:
return os


# Load the YAML configuration
with open("config/hotspot.yml", "r") as file:
config = yaml.safe_load(file)

# Iterate through configurations
for configuration in config["configurations"]:
directory = configuration["directory"]
architectures = configuration["architectures"]
os_name = configuration["os"]
os_family = osFamilyHelper(os_name)
base_image = configuration["image"]
deprecated = configuration.get("deprecated", None)
versions = configuration.get(
"versions", config["supported_distributions"]["Versions"]
)

# Define the path for the template based on OS
template_name = f"{os_name}.Dockerfile.j2"
template = env.get_template(template_name)

# Create output directories if they don't exist
for version in versions:
# if deprecated is set and version is greater than or equal to deprecated, skip
if deprecated and version >= deprecated:
continue
for image_type in ["jdk", "jre"]:
output_directory = os.path.join(str(version), image_type, directory)
os.makedirs(output_directory, exist_ok=True)

# Fetch latest release for version from Adoptium API
url = f"https://api.adoptium.net/v3/assets/feature_releases/{version}/ga?page=0&image_type={image_type}&page_size=1&vendor=eclipse"
response = requests.get(url, headers=headers)
response.raise_for_status()
data = response.json()

release = response.json()[0]

# Extract the version number from the release name
openjdk_version = release["release_name"]

# Generate the data for each architecture
arch_data = {}

for binary in release["binaries"]:
if (
binary["architecture"] in architectures
and binary["os"] == os_family
):
if os_family == "windows":
# Windows only has x64 binaries
copy_from = openjdk_version.replace(
"jdk", ""
) # jdk8u292-b10 -> 8u292-b10
if version != 8:
copy_from = copy_from.replace("-", "").replace(
"+", "_"
) # 11.0.11+9 -> 11.0.11_9
copy_from = f"{copy_from}-{image_type}-windowsservercore-{base_image.split(':')[1]}"
arch_data = {
"download_url": binary["installer"]["link"],
"checksum": binary["installer"]["checksum"],
"copy_from": copy_from,
}
# Iterate through OS families and then configurations
for os_family, configurations in config["configurations"].items():
for configuration in configurations:
directory = configuration["directory"]
architectures = configuration["architectures"]
os_name = configuration["os"]
base_image = configuration["image"]
deprecated = configuration.get("deprecated", None)
versions = configuration.get(
"versions", config["supported_distributions"]["Versions"]
)

# Define the path for the template based on OS
template_name = f"{os_name}.Dockerfile.j2"
template = env.get_template(template_name)

# Create output directories if they don't exist
for version in versions:
# if deprecated is set and version is greater than or equal to deprecated, skip
if deprecated and version >= deprecated:
continue
print("Generating Dockerfiles for", base_image, "-", version)
for image_type in ["jdk", "jre"]:
output_directory = os.path.join(str(version), image_type, directory)
os.makedirs(output_directory, exist_ok=True)

# Fetch latest release for version from Adoptium API
url = f"https://api.adoptium.net/v3/assets/feature_releases/{version}/ga?page=0&image_type={image_type}&page_size=1&vendor=eclipse"
response = requests.get(url, headers=headers)
response.raise_for_status()
data = response.json()

release = response.json()[0]

# Extract the version number from the release name
openjdk_version = release["release_name"]

# Generate the data for each architecture
arch_data = {}

for binary in release["binaries"]:
if (
binary["architecture"] in architectures
and binary["os"] == os_family
):
if os_family == "windows":
# Windows only has x64 binaries
copy_from = openjdk_version.replace(
"jdk", ""
) # jdk8u292-b10 -> 8u292-b10
if version != 8:
copy_from = copy_from.replace("-", "").replace(
"+", "_"
) # 11.0.11+9 -> 11.0.11_9
copy_from = f"{copy_from}-{image_type}-windowsservercore-{base_image.split(':')[1]}"
arch_data = {
"download_url": binary["installer"]["link"],
"checksum": binary["installer"]["checksum"],
"copy_from": copy_from,
}
else:
arch_data[archHelper(binary["architecture"], os_family)] = {
"download_url": binary["package"]["link"],
"checksum": binary["package"]["checksum"],
}

else:
arch_data[archHelper(binary["architecture"], os_family)] = {
"download_url": binary["package"]["link"],
"checksum": binary["package"]["checksum"],
}

# Generate Dockerfile for each architecture
rendered_dockerfile = template.render(
base_image=base_image,
image_type=image_type,
java_version=openjdk_version,
version=version,
arch_data=arch_data,
os_family=os_family,
os=os_name,
)
continue

# Save the rendered Dockerfile
with open(
os.path.join(output_directory, "Dockerfile"), "w"
) as out_file:
out_file.write(rendered_dockerfile)
# If arch_data is empty, skip updating the dockerfile
if arch_data.__len__() == 0:
continue

# Copy entrypoint.sh to output directory
entrypoint_path = os.path.join(
"docker_templates", "scripts", f"entrypoint.{os_name}.sh"
# Sort arch_data by key
arch_data = dict(sorted(arch_data.items()))

# Generate Dockerfile for each architecture
rendered_dockerfile = template.render(
base_image=base_image,
image_type=image_type,
java_version=openjdk_version,
version=version,
arch_data=arch_data,
os_family=os_family,
os=os_name,
)

print("Writing Dockerfile to", output_directory)
# Save the rendered Dockerfile
with open(
os.path.join(output_directory, "Dockerfile"), "w"
) as out_file:
out_file.write(rendered_dockerfile)

# Copy entrypoint.sh to output directory
entrypoint_path = os.path.join(
"docker_templates", "scripts", f"entrypoint.{os_name}.sh"
)

if os.path.exists(entrypoint_path):
os.system(
f"cp {entrypoint_path} {os.path.join(output_directory, 'entrypoint.sh')}"
)

if os.path.exists(entrypoint_path):
os.system(
f"cp {entrypoint_path} {os.path.join(output_directory, 'entrypoint.sh')}"
)

else:
continue

print("Dockerfiles generated successfully!")
53 changes: 0 additions & 53 deletions test_generate_dockerfiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,59 +16,6 @@

from jinja2 import Environment, FileSystemLoader

import generate_dockerfiles


class TestHelperFunctions(unittest.TestCase):
def test_archHelper(self):
test_data = [
("aarch64", "some-os", "aarch64|arm64"),
("ppc64le", "some-os", "ppc64el|powerpc:common64"),
("s390x", "some-os", "s390x|s390:64-bit"),
("arm", "some-os", "armhf|arm"),
("x64", "alpine-linux", "amd64|x86_64"),
("x64", "ubuntu", "amd64|i386:x86-64"),
("random-arch", "some-os", "random-arch"),
]

for arch, os_family, expected in test_data:
self.assertEqual(generate_dockerfiles.archHelper(arch, os_family), expected)

def test_osFamilyHelper(self):
test_data = [
("ubuntu", "linux"),
("centos", "linux"),
("ubi9-minimal", "linux"),
("nanoserver", "windows"),
("servercore", "windows"),
("random-os", "random-os"),
]

for os_name, expected in test_data:
self.assertEqual(generate_dockerfiles.osFamilyHelper(os_name), expected)

@patch("requests.get")
def test_fetch_latest_release(self, mock_get):
# Mocking the request.get call
mock_response = Mock()
mock_response.raise_for_status.return_value = None
mock_response.json.return_value = [{"key": "value"}]
mock_get.return_value = mock_response

url = "https://api.adoptium.net/v3/assets/feature_releases/some_version/ga?page=0&image_type=some_type&page_size=1&vendor=eclipse"
response = generate_dockerfiles.requests.get(
url, headers=generate_dockerfiles.headers
)
data = response.json()
self.assertIn("key", data[0])
self.assertEqual(data[0]["key"], "value")

@patch("builtins.open", new_callable=mock_open, read_data="configurations: []")
def test_load_config(self, mock_file):
with open("config/hotspot.yml", "r") as file:
config = generate_dockerfiles.yaml.safe_load(file)
self.assertIn("configurations", config)


class TestJinjaRendering(unittest.TestCase):
def setUp(self):
Expand Down

0 comments on commit a4b2525

Please sign in to comment.