Skip to content

Commit

Permalink
Automatically update builds in Bodhi update created from a sidetag (#…
Browse files Browse the repository at this point in the history 
…2549)

Automatically update builds in Bodhi update created from a sidetag

Fixes #2497.
Merge after packit/specfile#416.

Reviewed-by: Laura Barcziová
  • Loading branch information
@softwarefactory-project-zuul
softwarefactory-project-zuul[bot] authored Sep 30, 2024
2 parents aab45bf + 8d568d5 commit 0f6c6e6
Show file tree
Hide file tree
Showing 10 changed files with 277 additions and 213 deletions.
14 changes: 14 additions & 0 deletions packit_service/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2350,6 +2350,20 @@ def get_all_projects(cls) -> Set["GitProjectModel"]:
projects = [branch.project for branch in project_event_branches]
return set(projects)

@classmethod
def get_first_successful_by_sidetag(
cls, sidetag: str
) -> Optional["BodhiUpdateTargetModel"]:
with sa_session_transaction() as session:
return (
session.query(BodhiUpdateTargetModel)
.filter(
BodhiUpdateTargetModel.status == "success",
BodhiUpdateTargetModel.sidetag == sidetag,
)
.first()
)


class BodhiUpdateGroupModel(ProjectAndEventsConnector, GroupModel, Base):
__tablename__ = "bodhi_update_groups"
Expand Down
144 changes: 36 additions & 108 deletions packit_service/worker/handlers/bodhi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,12 @@
import logging
from datetime import datetime
from os import getenv
from typing import List, Tuple, Type, Optional
from typing import Tuple, Type, Optional

from celery import Task

from packit.config import JobConfig, JobType, PackageConfig, Deployment
from packit.exceptions import PackitException
from packit.utils.koji_helper import KojiHelper
from specfile.utils import NEVR
from packit_service.config import ServiceConfig
from packit_service.constants import (
MSG_RETRIGGER,
Expand All @@ -28,8 +26,6 @@
PipelineModel,
BodhiUpdateTargetModel,
KojiBuildTargetModel,
SidetagGroupModel,
SidetagModel,
)
from packit_service.worker.checker.abstract import Checker
from packit_service.worker.checker.bodhi import (
Expand All @@ -46,6 +42,7 @@
IssueCommentGitlabEvent,
)
from packit_service.worker.events.koji import KojiBuildEvent
from packit_service.worker.helpers.sidetag import SidetagHelper
from packit_service.worker.handlers.abstract import (
TaskName,
configured_as,
Expand Down Expand Up @@ -76,98 +73,11 @@
logger = logging.getLogger(__name__)


class SidetagHelper:
_koji_helper: Optional[KojiHelper] = None
_sidetag_group: Optional[SidetagGroupModel] = None

def __init__(self, job_config: JobConfig) -> None:
self.job_config = job_config

@property
def koji_helper(self) -> KojiHelper:
if not self._koji_helper:
self._koji_helper = KojiHelper()
return self._koji_helper

@property
def sidetag_group(self) -> SidetagGroupModel:
if not self._sidetag_group:
self._sidetag_group = SidetagGroupModel.get_by_name(
self.job_config.sidetag_group
)
return self._sidetag_group

def get_builds_in_sidetag(self, dist_git_branch: str) -> Tuple[str, List[str]]:
"""
Gets the latest builds of required packages from a sidetag identified
by the configured sidetag group and the specified dist-git branch.
Args:
dist_git_branch: dist-git branch.
Returns:
A tuple where the first element is the name of the sidetag
and the second element is a list of NVRs.
Raises:
PackitException if a sidetag was not found or if job dependencies
are not satisfied.
"""
sidetag = self.sidetag_group.get_sidetag_by_target(dist_git_branch)
if not sidetag:
raise PackitException(
f"No sidetag found for {self.sidetag_group.name} and {dist_git_branch}"
)

builds = self.koji_helper.get_builds_in_tag(sidetag.koji_name)
tagged_packages = {b["package_name"] for b in builds}

# check if dependencies are satisfied within the sidetag
dependencies = set(self.job_config.dependencies or [])
dependencies.add(self.job_config.downstream_package_name) # include self
if not dependencies <= tagged_packages:
missing = dependencies - tagged_packages
raise PackitException(f"Missing dependencies for Bodhi update: {missing}")

nvrs = []
for package in dependencies:
latest_stable_nvr = self.koji_helper.get_latest_stable_nvr(
package, dist_git_branch, include_candidate=True
)
latest_build_in_sidetag = max(
(b for b in builds if b["package_name"] == package),
key=lambda b: NEVR.from_string(b["nvr"]),
)
# exclude NVRs that are already in stable or candidate tags - if a build
# has been manually tagged into the sidetag to satisfy dependencies,
# we don't want it in the update
if NEVR.from_string(latest_build_in_sidetag["nvr"]) > NEVR.from_string(
latest_stable_nvr
):
nvrs.append(latest_build_in_sidetag["nvr"])

return sidetag.koji_name, nvrs

def remove_sidetag(self, koji_name: str) -> None:
"""Removes the specified sidetag."""
self.koji_helper.remove_sidetag(koji_name)
if sidetag := SidetagModel.get_by_koji_name(koji_name):
sidetag.delete()


class BodhiUpdateHandler(
RetriableJobHandler, PackitAPIWithDownstreamMixin, GetKojiBuildData
):
topic = "org.fedoraproject.prod.buildsys.build.state.change"

_sidetag_helper: Optional[SidetagHelper] = None

@property
def sidetag_helper(self) -> SidetagHelper:
if not self._sidetag_helper:
self._sidetag_helper = SidetagHelper(self.job_config)
return self._sidetag_helper

def __init__(
self,
package_config: PackageConfig,
Expand All @@ -194,8 +104,21 @@ def run(self) -> TaskResults:
errors = {}
for target_model in group.grouped_targets:
try:
existing_alias = None
if target_model.sidetag:
# get update alias from previous run(s) from the same sidetag (if any)
if model := BodhiUpdateTargetModel.get_first_successful_by_sidetag(
target_model.sidetag
):
existing_alias = model.alias

logger.debug(
f"Create update for dist-git branch: {target_model.target} "
(
f"Edit update {existing_alias} "
if existing_alias
else "Create update "
)
+ f"for dist-git branch: {target_model.target} "
f"and nvrs: {target_model.koji_nvrs}"
+ (
f" from sidetag: {target_model.sidetag}."
Expand All @@ -208,21 +131,13 @@ def run(self) -> TaskResults:
update_type="enhancement",
koji_builds=target_model.koji_nvrs.split(), # it accepts NVRs, not build IDs
sidetag=target_model.sidetag,
alias=existing_alias,
)
if not result:
# update was already created
target_model.set_status("skipped")
continue

if target_model.sidetag:
# remove the sidetag now; Bodhi would remove it for us
# when the update hits stable, but we would be blocked
# from creating new updates until that happens
logger.debug(f"Removing sidetag {target_model.sidetag}")
# we need Kerberos ticket to remove a sidetag
self.packit_api.init_kerberos_ticket()
self.sidetag_helper.remove_sidetag(target_model.sidetag)

alias, url = result
target_model.set_status("success")
target_model.set_alias(alias)
Expand Down Expand Up @@ -290,17 +205,30 @@ def _get_or_create_bodhi_update_group_model(self) -> BodhiUpdateGroupModel:
group = BodhiUpdateGroupModel.create(run_model)

for koji_build_data in self:
sidetag = builds = None
if self.job_config.sidetag_group:
sidetag, builds = self.sidetag_helper.get_builds_in_sidetag(
koji_build_data.dist_git_branch
sidetag = SidetagHelper.get_sidetag(
self.job_config.sidetag_group, koji_build_data.dist_git_branch
)
# check if dependencies are satisfied within the sidetag
dependencies = set(self.job_config.dependencies or [])
dependencies.add(
self.job_config.downstream_package_name # include self
)
if missing_dependencies := sidetag.get_missing_dependencies(
dependencies
):
raise PackitException(
f"Missing dependencies for Bodhi update: {missing_dependencies}"
)
builds = " ".join(
str(b) for b in sidetag.get_builds_suitable_for_update(dependencies)
)
else:
sidetag = builds = None

BodhiUpdateTargetModel.create(
target=koji_build_data.dist_git_branch,
koji_nvrs=koji_build_data.nvr if not builds else " ".join(builds),
sidetag=sidetag,
koji_nvrs=koji_build_data.nvr if not builds else builds,
sidetag=sidetag.koji_name if sidetag else None,
status="queued",
bodhi_update_group=group,
)
Expand Down
87 changes: 16 additions & 71 deletions packit_service/worker/handlers/distgit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
PackitDownloadFailedException,
ReleaseSkippedPackitException,
)
from packit.utils.koji_helper import KojiHelper
from packit_service import sentry_integration
from packit_service.config import PackageConfigGetter, ServiceConfig
from packit_service.constants import (
Expand All @@ -45,8 +44,6 @@
KojiBuildTargetModel,
PipelineModel,
KojiBuildGroupModel,
SidetagGroupModel,
SidetagModel,
)
from packit_service.service.urls import (
get_propose_downstream_info_url,
Expand Down Expand Up @@ -90,6 +87,7 @@
RetriableJobHandler,
)
from packit_service.worker.handlers.mixin import GetProjectToSyncMixin
from packit_service.worker.helpers.sidetag import SidetagHelper
from packit_service.worker.helpers.sync_release.propose_downstream import (
ProposeDownstreamJobHelper,
)
Expand Down Expand Up @@ -716,8 +714,6 @@ class AbstractDownstreamKojiBuildHandler(
topic = "org.fedoraproject.prod.pagure.git.receive"
task_name = TaskName.downstream_koji_build

_koji_helper: Optional[KojiHelper] = None

def __init__(
self,
package_config: PackageConfig,
Expand All @@ -737,12 +733,6 @@ def __init__(
self._packit_api = None
self._koji_group_model_id = koji_group_model_id

@property
def koji_helper(self):
if not self._koji_helper:
self._koji_helper = KojiHelper()
return self._koji_helper

@staticmethod
def get_checkers() -> Tuple[Type[Checker], ...]:
return (
Expand Down Expand Up @@ -781,13 +771,6 @@ def get_branches(self) -> List[str]:
def run(self) -> TaskResults:
errors = {}

if self.job_config.sidetag_group:
sidetag_group = SidetagGroupModel.get_or_create(
self.job_config.sidetag_group
)
else:
sidetag_group = None

group = self._get_or_create_koji_group_model()
for koji_build_model in group.grouped_targets:
branch = koji_build_model.target
Expand All @@ -804,32 +787,22 @@ def run(self) -> TaskResults:
koji_build_model.set_status("pending")

try:
if sidetag_group:
sidetag = SidetagModel.get_or_create(sidetag_group, branch)
if not sidetag.koji_name or not self.koji_helper.get_tag_info(
sidetag.koji_name
sidetag = None
if self.job_config.sidetag_group:
# we need Kerberos ticket to create a new sidetag
self.packit_api.init_kerberos_ticket()
sidetag = SidetagHelper.get_or_create_sidetag(
self.job_config.sidetag_group, branch
)
# skip submitting build for a branch if dependencies
# are not satisfied within a sidetag
dependencies = set(self.job_config.dependencies or [])
if missing_dependencies := sidetag.get_missing_dependencies(
dependencies
):
# we need Kerberos ticket to create a new sidetag
self.packit_api.init_kerberos_ticket()
tag_info = self.koji_helper.create_sidetag(branch)
if not tag_info:
raise PackitException(
f"Failed to create sidetag for {branch}"
)
sidetag.set_koji_name(tag_info["name"])
else:
sidetag = None

# skip submitting build for a branch if dependencies
# are not satisfied within a sidetag
if sidetag and self.job_config.dependencies:
builds = self.koji_helper.get_builds_in_tag(sidetag.koji_name)
tagged_packages = {b["package_name"] for b in builds}
if not set(self.job_config.dependencies) <= tagged_packages:
missing = set(self.job_config.dependencies) - tagged_packages
logger.debug(
f"Skipping downstream Koji build for branch {branch}, "
f"missing dependencies: {missing}"
f"missing dependencies: {missing_dependencies}"
)
koji_build_model.set_status("skipped")
continue
Expand Down Expand Up @@ -1041,44 +1014,16 @@ class TagIntoSidetagHandler(
):
task_name = TaskName.tag_into_sidetag

_koji_helper: Optional[KojiHelper] = None

@property
def koji_helper(self):
if not self._koji_helper:
self._koji_helper = KojiHelper()
return self._koji_helper

@staticmethod
def get_checkers() -> Tuple[Type[Checker], ...]:
return (PermissionOnDistgit,)

def run_for_branch(self, job_config: JobConfig, branch: str) -> None:
sidetag_group = SidetagGroupModel.get_or_create(job_config.sidetag_group)
sidetag = SidetagModel.get_or_create(sidetag_group, branch)
# we need Kerberos ticket to tag a build into sidetag
# and to create a new sidetag (if needed)
self.packit_api.init_kerberos_ticket()
if not sidetag.koji_name or not self.koji_helper.get_tag_info(
sidetag.koji_name
):
tag_info = self.koji_helper.create_sidetag(branch)
if not tag_info:
logger.error(f"Failed to create sidetag for {branch}")
return
sidetag.set_koji_name(tag_info["name"])
if not (
nvr := self.koji_helper.get_latest_stable_nvr(
job_config.downstream_package_name, branch
)
):
logger.debug(
"Failed to get the latest stable build "
f"of {job_config.downstream_package_name} for {branch}"
)
return
logger.debug(f"Tagging {nvr} into {sidetag.koji_name}")
self.koji_helper.tag_build(nvr, sidetag.koji_name)
sidetag = SidetagHelper.get_or_create_sidetag(job_config.sidetag_group, branch)
sidetag.tag_latest_stable_build(job_config.downstream_package_name)

def run(self) -> TaskResults:
comment = self.data.event_dict.get("comment")
Expand Down
Loading

0 comments on commit 0f6c6e6

Please sign in to comment.