Skip to content

Commit 7185d02

Browse files
alielaliel
committed
Update CPU and memory details by switching to lshw method instead of cpuinfo
Add CPU information (model, vendor, frequency) and memory details (clock, size, type) to API
1 parent a04783d commit 7185d02

10 files changed

+142
-15
lines changed

‎.github/workflows/code-quality.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
run: |
1717
sudo apt-get update
1818
sudo apt-get -y upgrade
19-
sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema
19+
sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-nftables python3-jsonschema
2020
pip install --upgrade typing-extensions types-PyYAML
2121
2222
- name: Install required Python packages

‎docker/vm_supervisor-dev.dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ FROM debian:bullseye
55
RUN apt-get update && apt-get -y upgrade && apt-get install -y \
66
sudo acl curl squashfs-tools git \
77
python3 python3-aiohttp python3-alembic python3-msgpack python3-pip python3-aiodns python3-aioredis\
8-
python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging python3-cpuinfo ndppd \
8+
python3-nftables python3-psutil python3-setproctitle python3-sqlalchemy python3-packaging ndppd \
99
&& rm -rf /var/lib/apt/lists/*
1010

1111
RUN useradd jailman

‎packaging/aleph-vm/DEBIAN/control

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ Version: 0.1.8
33
Architecture: all
44
Maintainer: Aleph.im
55
Description: Aleph.im VM execution engine
6-
Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-cpuinfo,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs
6+
Depends: python3,python3-pip,python3-aiohttp,python3-msgpack,python3-aiodns,python3-alembic,python3-sqlalchemy,python3-setproctitle,redis,python3-aioredis,python3-psutil,sudo,acl,curl,systemd-container,squashfs-tools,debootstrap,python3-packaging,python3-nftables,python3-jsonschema,cloud-image-utils,ndppd,python3-yaml,python3-dotenv,python3-schedule,qemu-system-x86,qemu-utils,python3-systemd,python3-dbus,btrfs-progs,lshw
77
Section: aleph-im
88
Priority: Extra

‎packaging/requirements-debian-11.txt

-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ multidict==5.1.0
1717
git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
1818
packaging==20.9
1919
psutil==5.8.0
20-
py-cpuinfo==5.0.0
2120
pycares==3.1.1
2221
pyparsing==2.4.7
2322
pyrsistent==0.15.5

‎packaging/requirements-ubuntu-20.04.txt

-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ multidict==4.7.3
1818
git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
1919
packaging==20.3
2020
psutil==5.5.1
21-
py-cpuinfo==5.0.0
2221
pycares==3.1.1
2322
PyGObject==3.36.0
2423
pyparsing==2.4.6

‎packaging/requirements-ubuntu-22.04.txt

-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ multidict==5.1.0
2121
git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py
2222
packaging==21.3
2323
psutil==5.9.0
24-
py-cpuinfo==5.0.0
2524
pycares==4.1.2
2625
PyGObject==3.42.1
2726
pyparsing==2.4.7

‎pyproject.toml

-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ dependencies = [
3535
"sentry-sdk==1.31.0",
3636
"aioredis==1.3.1",
3737
"psutil==5.9.5",
38-
"py-cpuinfo==9.0.0",
3938
"schedule==1.2.1",
4039
"nftables @ git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py",
4140
"msgpack==1.0.7",

‎src/aleph/vm/orchestrator/machine.py

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import json
2+
import re
3+
import subprocess
4+
from functools import lru_cache
5+
6+
import psutil
7+
8+
9+
@lru_cache
10+
def get_hardware_info():
11+
lshw = subprocess.Popen(["lshw", "-sanitize", "-json"], stdout=subprocess.PIPE, shell=False)
12+
output, _ = lshw.communicate()
13+
data = json.loads(output)
14+
15+
hw_info = {}
16+
17+
for hw in data["children"][0]["children"]:
18+
if hw["id"] == "cpu":
19+
hw_info["cpu"] = hw
20+
elif hw["class"] == "memory" and hw["id"] == "memory":
21+
hw_info["memory"] = hw
22+
23+
return hw_info
24+
25+
26+
@lru_cache
27+
def get_cpu_info():
28+
hw = get_hardware_info()
29+
30+
cpu_info = hw["cpu"]
31+
architecture = cpu_info["width"]
32+
33+
if "x86_64" in cpu_info["capabilities"] or "x86-64" in cpu_info["capabilities"]:
34+
architecture = "x86_64"
35+
elif "arm64" in cpu_info["capabilities"] or "arm-64" in cpu_info["capabilities"]:
36+
architecture = "arm64"
37+
38+
vendor = cpu_info["vendor"]
39+
# lshw vendor implementation => https://github.com/lyonel/lshw/blob/15e4ca64647ad119b69be63274e5de2696d3934f/src/core/cpuinfo.cc#L308
40+
41+
if "Intel Corp" in vendor:
42+
vendor = "GenuineIntel"
43+
elif "Advanced Micro Devices [AMD]" in vendor:
44+
vendor = "AuthenticAMD"
45+
46+
return {
47+
"architecture": architecture,
48+
"vendor": vendor,
49+
"model": cpu_info["product"],
50+
"frequency": cpu_info["capacity"],
51+
"count": psutil.cpu_count(),
52+
}
53+
54+
55+
@lru_cache
56+
def get_memory_info():
57+
hw = get_hardware_info()
58+
mem_info = hw["memory"]
59+
60+
memory_type = ""
61+
memory_clock = ""
62+
63+
for bank in mem_info["children"]:
64+
memory_clock = bank["clock"]
65+
try:
66+
memory_type = re.search("(DDR[2-6])", bank["description"]).group(0)
67+
break
68+
except:
69+
pass
70+
71+
return {
72+
"size": mem_info["size"],
73+
"units": mem_info["units"],
74+
"type": memory_type,
75+
"clock": memory_clock,
76+
"clock_units": "Hz",
77+
}

‎src/aleph/vm/orchestrator/resources.py

+60-6
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,14 @@
33
from functools import lru_cache
44
from typing import Optional
55

6-
import cpuinfo
76
import psutil
87
from aiohttp import web
8+
from aleph.vm.conf import settings
9+
from aleph.vm.orchestrator.machine import get_cpu_info, get_memory_info
910
from aleph_message.models import ItemHash
1011
from aleph_message.models.execution.environment import CpuProperties
1112
from pydantic import BaseModel, Field
1213

13-
from aleph.vm.conf import settings
14-
1514

1615
class Period(BaseModel):
1716
datetime: datetime
@@ -76,18 +75,66 @@ class MachineUsage(BaseModel):
7675
active: bool = True
7776

7877

78+
class ExtendedCpuProperties(CpuProperties):
79+
"""CPU properties."""
80+
81+
model: Optional[str] = Field(default=None, description="CPU model")
82+
frequency: Optional[str] = Field(default=None, description="CPU frequency")
83+
count: Optional[str] = Field(default=None, description="CPU count")
84+
85+
86+
87+
class MemoryProperties(BaseModel):
88+
"""MEMORY properties."""
89+
90+
size: Optional[str] = Field(default=None, description="Memory size")
91+
units: Optional[str] = Field(default=None, description="Memory size units")
92+
type: Optional[str] = Field(default=None, description="Memory type")
93+
clock: Optional[str] = Field(default=None, description="Memory clock")
94+
clock_units: Optional[str] = Field(default=None, description="Memory clock units")
95+
96+
97+
class MachineCapability(BaseModel):
98+
cpu: ExtendedCpuProperties
99+
memory: MemoryProperties
100+
101+
79102
@lru_cache
80103
def get_machine_properties() -> MachineProperties:
81104
"""Fetch machine properties such as architecture, CPU vendor, ...
82105
These should not change while the supervisor is running.
83106
84107
In the future, some properties may have to be fetched from within a VM.
85108
"""
86-
cpu_info = cpuinfo.get_cpu_info() # Slow
109+
110+
cpu_info = get_cpu_info()
87111
return MachineProperties(
88112
cpu=CpuProperties(
89-
architecture=cpu_info["raw_arch_string"],
90-
vendor=cpu_info["vendor_id"],
113+
architecture=cpu_info["architecture"],
114+
vendor=cpu_info["vendor"],
115+
),
116+
)
117+
118+
119+
@lru_cache
120+
def get_machine_capability() -> MachineCapability:
121+
cpu_info = get_cpu_info()
122+
mem_info = get_memory_info()
123+
124+
return MachineCapability(
125+
cpu=ExtendedCpuProperties(
126+
architecture=cpu_info["architecture"],
127+
vendor=cpu_info["vendor"],
128+
model=cpu_info["model"],
129+
frequency=cpu_info["frequency"],
130+
count=cpu_info["count"],
131+
),
132+
memory=MemoryProperties(
133+
size=mem_info["size"],
134+
units=mem_info["units"],
135+
type=mem_info["type"],
136+
clock=mem_info["clock"],
137+
clock_units=mem_info["clock_units"],
91138
),
92139
)
93140

@@ -119,6 +166,13 @@ async def about_system_usage(_: web.Request):
119166
return web.json_response(text=usage.json(exclude_none=True), headers={"Access-Control-Allow-Origin:": "*"})
120167

121168

169+
async def about_capability(_: web.Request):
170+
"""Public endpoint to expose information about the CRN capability."""
171+
172+
capability: MachineCapability = get_machine_capability()
173+
return web.json_response(text=capability.json(exclude_none=False), headers={"Access-Control-Allow-Origin:": "*"})
174+
175+
122176
class Allocation(BaseModel):
123177
"""An allocation is the set of resources that are currently allocated on this orchestrator.
124178
It contains the item_hashes of all persistent VMs, instances, on-demand VMs and jobs.

‎src/aleph/vm/orchestrator/supervisor.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,12 @@
1414
from typing import Callable
1515

1616
from aiohttp import web
17-
1817
from aleph.vm.conf import settings
1918
from aleph.vm.pool import VmPool
2019
from aleph.vm.version import __version__
2120

2221
from .metrics import create_tables, setup_engine
23-
from .resources import about_system_usage
22+
from .resources import about_capability, about_system_usage
2423
from .tasks import (
2524
start_payment_monitoring_task,
2625
start_watch_for_messages_task,
@@ -93,6 +92,7 @@ async def allow_cors_on_endpoint(request: web.Request):
9392
web.get("/about/executions/records", about_execution_records),
9493
web.get("/about/usage/system", about_system_usage),
9594
web.get("/about/config", about_config),
95+
web.get("/about/capability", about_capability),
9696
# /control APIs are used to control the VMs and access their logs
9797
web.post("/control/allocations", update_allocations),
9898
web.post("/control/allocation/notify", notify_allocation),

0 commit comments

Comments
 (0)