Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Ruff as a code checker #222

Merged
merged 2 commits into from
Jul 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# This is the configuration file for Dependabot. You can find configuration information below.
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
# Note: Dependabot has a configurable max open PR limit of 5

version: 2
updates:

# Maintain dependencies for our GitHub Actions
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"
labels:
- "dependencies"
11 changes: 7 additions & 4 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11"]

steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: Initialize CodeQL
uses: github/codeql-action/init@v2
Expand All @@ -35,10 +35,13 @@ jobs:
uses: github/codeql-action/analyze@v2

- name: Setup Python
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Pre Commit Checks
uses: pre-commit/[email protected]

- name: Setup Temp Directory
run: mkdir broker_dir

Expand All @@ -47,7 +50,7 @@ jobs:
BROKER_DIRECTORY: "${{ github.workspace }}/broker_dir"
run: |
pip install -U pip
pip install -U .[test,docker]
pip install -U .[dev,docker]
ls -l "$BROKER_DIRECTORY"
broker --version
pytest -v tests/ --ignore tests/functional
4 changes: 2 additions & 2 deletions .github/workflows/python-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: PythonPackage

on:
push:
tags:
tags:
- "*"

jobs:
Expand All @@ -12,7 +12,7 @@ jobs:
strategy:
matrix:
# build/push in lowest support python version
python-version: [ 3.9 ]
python-version: [ 3.10 ]

steps:
- uses: actions/checkout@v2
Expand Down
12 changes: 12 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# configuration for pre-commit git hooks
repos:
- repo: https://github.com/psf/black
rev: 23.3.0
hooks:
- id: black
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.0.277
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
1 change: 0 additions & 1 deletion MANIFEST.in

This file was deleted.

5 changes: 2 additions & 3 deletions broker/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
from broker.broker import Broker

VMBroker = Broker
"""Shortcuts for the broker module."""
from broker.broker import Broker # noqa: F401
1 change: 1 addition & 0 deletions broker/binds/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Binds provide interfaces between a provider's interface and the Broker Provider class."""
64 changes: 43 additions & 21 deletions broker/binds/beaker.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
"""A wrapper around the Beaker CLI."""
import json
from pathlib import Path
import subprocess
import time
from logzero import logger
from pathlib import Path
from xml.etree import ElementTree as ET

from logzero import logger

from broker import helpers
from broker.exceptions import BeakerBindError


def _elementree_to_dict(etree):
"""Converts an ElementTree object to a dictionary"""
"""Convert an ElementTree object to a dictionary."""
data = {}
if etree.attrib:
data.update(etree.attrib)
Expand All @@ -32,12 +35,14 @@ def _curate_job_info(job_info_dict):
# "reservation_id": "current_reservation/recipe_id",
"whiteboard": "whiteboard/text",
"hostname": "recipeSet/recipe/system",
"distro": "recipeSet/recipe/distro"
"distro": "recipeSet/recipe/distro",
}
return helpers.dict_from_paths(job_info_dict, curated_info)


class BeakerBind:
"""A bind class providing a basic interface to the Beaker CLI."""

def __init__(self, hub_url, auth="krbv", **kwargs):
self.hub_url = hub_url
self._base_args = ["--insecure", f"--hub={self.hub_url}"]
Expand All @@ -54,6 +59,11 @@ def __init__(self, hub_url, auth="krbv", **kwargs):
self.__dict__.update(kwargs)

def _exec_command(self, *cmd_args, **cmd_kwargs):
"""Execute a beaker command and return the result.

cmd_args: Expanded into feature flags for the beaker command
cmd_kwargs: Expanded into args and values for the beaker command
"""
raise_on_error = cmd_kwargs.pop("raise_on_error", True)
exec_cmd, cmd_args = ["bkr"], list(cmd_args)
# check through kwargs and if any are True add to cmd_args
Expand Down Expand Up @@ -82,14 +92,13 @@ def _exec_command(self, *cmd_args, **cmd_kwargs):
)
if result.status != 0 and raise_on_error:
raise BeakerBindError(
f"Beaker command failed:\n"
f"Command={' '.join(exec_cmd)}\n"
f"Result={result}",
f"Beaker command failed:\nCommand={' '.join(exec_cmd)}\nResult={result}",
)
logger.debug(f"Beaker command result: {result.stdout}")
return result

def job_submit(self, job_xml, wait=False):
"""Submit a job to Beaker and optionally wait for it to complete."""
# wait behavior seems buggy to me, so best to avoid it
if not Path(job_xml).exists():
raise FileNotFoundError(f"Job XML file {job_xml} not found")
Expand All @@ -102,36 +111,41 @@ def job_submit(self, job_xml, wait=False):
return line.split("'")[1].replace("J:", "")

def job_watch(self, job_id):
"""Watch a job via the job-watch command. This can be buggy."""
job_id = f"J:{job_id}" if not job_id.startswith("J:") else job_id
return self._exec_command("job-watch", job_id)

def job_results(self, job_id, format="beaker-results-xml", pretty=False):
"""Get the results of a job in the specified format."""
job_id = f"J:{job_id}" if not job_id.startswith("J:") else job_id
return self._exec_command(
"job-results", job_id, format=format, prettyxml=pretty
)
return self._exec_command("job-results", job_id, format=format, prettyxml=pretty)

def job_clone(self, job_id, wait=False, **kwargs):
"""Clone a job by the specified job id."""
job_id = f"J:{job_id}" if not job_id.startswith("J:") else job_id
return self._exec_command("job-clone", job_id, wait=wait, **kwargs)

def job_list(self, *args, **kwargs):
"""List jobs matching the criteria specified by args and kwargs."""
return self._exec_command("job-list", *args, **kwargs)

def job_cancel(self, job_id):
"""Cancel a job by the specified job id."""
if not job_id.startswith("J:") and not job_id.startswith("RS:"):
job_id = f"J:{job_id}"
return self._exec_command("job-cancel", job_id)

def job_delete(self, job_id):
"""Delete a job by the specified job id."""
job_id = f"J:{job_id}" if not job_id.startswith("J:") else job_id
return self._exec_command("job-delete", job_id)

def system_release(self, system_id):
"""Release a system by the specified system id."""
return self._exec_command("system-release", system_id)

def system_list(self, **kwargs):
"""Due to the number of arguments, we will not validate before submitting
"""Due to the number of arguments, we will not validate before submitting.

Accepted arguments are:
available available to be used by this user
Expand All @@ -152,25 +166,30 @@ def system_list(self, **kwargs):
host-filter=NAME matching pre-defined host filter
"""
# convert the flags passed in kwargs to arguments
args = []
for key in {"available", "free", "removed", "mine"}:
if kwargs.pop(key, False):
args.append(f"--{key}")
args = [
f"--{key}" for key in ("available", "free", "removed", "mine") if kwargs.pop(key, False)
]
return self._exec_command("system-list", *args, **kwargs)

def user_systems(self): # to be used for inventory sync
result = self.system_list(mine=True, raise_on_error=False)
def user_systems(self):
"""Return a list of system ids owned by the current user.

This is used for inventory syncing against Beaker.
"""
result = self.system_list(mine=True, raise_on_error=False)
if result.status != 0:
return []
else:
return result.stdout.splitlines()

def system_details(self, system_id, format="json"):
"""Get details about a system by the specified system id."""
return self._exec_command("system-details", system_id, format=format)

def execute_job(self, job, max_wait="24h"):
"""Submit a job, periodically checking the status until it completes
then return a dictionary of the results.
"""Submit a job, periodically checking the status until it completes.

return: a dictionary of the results.
"""
if Path(job).exists(): # job xml path passed in
job_id = self.job_submit(job, wait=False)
Expand All @@ -187,10 +206,13 @@ def execute_job(self, job, max_wait="24h"):
raise BeakerBindError(f"Job {job_id} failed:\n{result}")
elif 'result="Warn"' in result.stdout:
res_dict = _elementree_to_dict(ET.fromstring(result.stdout))
raise BeakerBindError(f"Job {job_id} was resulted in a warning. Status: {res_dict['status']}")
raise BeakerBindError(
f"Job {job_id} was resulted in a warning. Status: {res_dict['status']}"
)
raise BeakerBindError(f"Job {job_id} did not complete within {max_wait}")

def system_details_curated(self, system_id):
"""Return a curated dictionary of system details."""
full_details = json.loads(self.system_details(system_id).stdout)
curated_details = {
"hostname": full_details["fqdn"],
Expand All @@ -216,7 +238,7 @@ def system_details_curated(self, system_id):
return curated_details

def jobid_from_system(self, system_hostname):
"""Return the job id for the current reservation on the system"""
"""Return the job id for the current reservation on the system."""
for job_id in json.loads(self.job_list(mine=True).stdout):
job_result = self.job_results(job_id, pretty=True)
job_detail = _curate_job_info(_elementree_to_dict(ET.fromstring(job_result.stdout)))
Expand Down
Loading