ENTERPRISE LINUX KERNEL BUILD TOOL FOR AMD/NVIDIA/INTEL GPU
THIS TOOL EXPECTS YOU TO HAVE A VERY RECENT VERSION OF PYTHON INSTALLED, AND ASKS YOU A FEW QUESTIONS BEFORE IT RUNS, LIKE WHAT COMPILER YOU WANT TO USE, BUT ALMOST EVERYTHING IS AUTOMATIC, I AM STILL TESTING WHETHER OR NOT THE DOWNLOADED SCRIPT WORKS OR NOT AT THIS TIME… BUT YOU CAN CLICK “download enterprise_linux_kernel_tool.py” AND IT WILL SAVE IT TO YOUR COMPUTER, AND YOU CAN SIMPLY RUN THE SCRIPT, ANSWER ABOUT 6 QUESTIONS AS TO WHETHER OR NOT YOU WANT ALL DRIVERS, AND A NATIVE CPU KERNEL, AND EVERYTHING ELSE IS CONFIGURED FOR YOU!
I’VE TESTED THE SCRIPT THAT IS COPIED FROM THE TEXT ON THIS PAGE FOR THE DOWNLOAD BUTTON, AND IT IS WORKING.
PYTHON 3.14.7 download enterprise_linux_kernel_tool.py
builds a kernel for enterprise linux for any gpu
#!/usr/bin/python3 -I
# -*- coding: utf-8 -*-
"""
enterprise_linux_kernel_tool.py
Pick a kernel from kernel.org, configure it for the machine it is run on, build
it, prove that out-of-tree (DKMS) modules still compile against it, and install
it without taking over the boot menu.
./enterprise_linux_kernel_tool.py interactive wizard
./enterprise_linux_kernel_tool.py kernels what kernel.org offers
./enterprise_linux_kernel_tool.py detect what this machine is
./enterprise_linux_kernel_tool.py doctor can this machine build?
./enterprise_linux_kernel_tool.py plan --kernel stable
./enterprise_linux_kernel_tool.py build --kernel 6.12 --modules trimmed
./enterprise_linux_kernel_tool.py install --source ~/src/linux-7.2.6
./enterprise_linux_kernel_tool.py dkms-check --source ~/src/linux-7.2.6
Standard library only, and the SYNTAX is held to Python 3.6 so RHEL 8's
platform interpreter can run it: no walrus, no `capture_output=`, no
`text=`, no dataclasses. The shebang pins /usr/bin/python3 -- the interpreter
the package manager itself depends on -- because "python3" on PATH is not
reliably CPython (on the machine this was written on it is a PyPy whose ssl
module segfaults).
THE ONE THING THIS TOOL EXISTS TO GET RIGHT, inherited from the shell script it
replaces: GPU drivers are MODULES. A built-in (=y) driver probes before the
root filesystem is mounted, so its firmware must already be in the initramfs --
but dracut and initramfs-tools pick firmware by looking at installed MODULES,
and a built-in driver is not one. Result: no display, no /dev/kfd. As modules
they also stay replaceable, which is what amdgpu-dkms and NVIDIA's stack need.
Licence: public domain / CC0. No warranty. It installs a kernel; read it first.
"""
import sys
if sys.version_info < (3, 6):
sys.exit("enterprise_linux_kernel_tool.py needs Python 3.6 or newer")
import argparse
import configparser
import fcntl
import glob
import gzip
import hashlib
import json
import os
import re
import shlex
import shutil
import signal
import subprocess
import tempfile
import threading
import time
import urllib.error
import urllib.request
TOOL = "enterprise_linux_kernel_tool"
VERSION = "0.1.0"
PLATFORM_PYTHON = "/usr/bin/python3"
KORG_RELEASES = "https://www.kernel.org/releases.json"
KORG_CDN = "https://cdn.kernel.org/pub/linux/kernel"
USER_AGENT = TOOL + "/" + VERSION
# ------------------------------------------------------------------------ output
class UI(object):
color = sys.stdout.isatty() and "NO_COLOR" not in os.environ
tty = sys.stdout.isatty()
def _c(code, text):
return "\033[%sm%s\033[0m" % (code, text) if UI.color else text
def out(text=""):
try:
print(text)
except UnicodeEncodeError: # a C locale meeting U+FFFD from errors="replace"
print(text.encode("ascii", "replace").decode("ascii"))
sys.stdout.flush()
def say(msg):
out("\n" + _c("1", "==> " + msg))
def info(msg):
out(" " + msg)
def ok(msg):
out(" " + _c("32", "ok ") + " " + msg)
def note(msg):
out(" -- " + msg)
def bad(msg):
out(" " + _c("31", "FAIL") + " " + msg)
def warn(msg):
# stdout, not stderr: with both redirected to one log file the two streams
# interleave out of order, and a warning that lands three sections late is
# worse than no warning.
out(_c("33", "warning:") + " " + msg)
class ToolError(Exception):
"""A failure with a message fit to show the user. No traceback."""
# ----------------------------------------------------------------------- process
def which(name, path=None):
return shutil.which(name, path=path)
def run(cmd, cwd=None, env=None, check=True, capture=True, stdin_data=None,
interactive=False, timeout=None):
"""subprocess.run with 3.6-safe arguments. stdin is /dev/null unless asked
for, so nothing downstream can sit waiting on a prompt nobody will see."""
kw = dict(cwd=cwd, env=env, universal_newlines=True, errors="replace",
timeout=timeout)
if capture and not interactive:
kw["stdout"] = subprocess.PIPE
kw["stderr"] = subprocess.STDOUT
if stdin_data is not None:
kw["input"] = stdin_data
elif not interactive:
kw["stdin"] = subprocess.DEVNULL
try:
p = subprocess.run(cmd, **kw)
except FileNotFoundError:
raise ToolError("command not found: %s" % cmd[0])
except subprocess.TimeoutExpired:
raise ToolError("timed out: %s" % " ".join(cmd))
if check and p.returncode != 0:
tail = (p.stdout or "").strip()[-2000:]
raise ToolError("command failed (exit %d): %s\n%s"
% (p.returncode, " ".join(cmd), tail))
return p
# Environment for every make invocation. An ALLOW-list: the traps that cost
# whole builds on real machines were all inherited variables nobody remembered
# exporting -- CC/CXX pointing at a private clang, LD_RUN_PATH baked into every
# host tool's RPATH, LOCALVERSION appended a second time by setlocalversion,
# PYTHON* steering kbuild's scripts into another interpreter.
ENV_KEEP = ("HOME", "USER", "LOGNAME", "TERM", "TMPDIR", "SHELL", "CCACHE_DIR")
SYSTEM_PATH = ["/usr/bin", "/usr/sbin", "/bin", "/sbin", "/usr/local/bin"]
def scrubbed_env(extra_path=None):
env = dict((k, os.environ[k]) for k in ENV_KEEP if k in os.environ)
env["ELKT_NO_SIGN"] = "1" # the tree's wrapper signs for OTHER callers; we do it ourselves
path = ([extra_path] if extra_path else []) + SYSTEM_PATH
env["PATH"] = ":".join(path)
env["LC_ALL"] = "C"
return env
def dkms_env(pkg_manager):
"""What an out-of-tree build sees when DKMS runs it: root, via sudo or a
package hook, with sudo's secure_path and none of the user's exports."""
path = ["/usr/sbin", "/usr/bin", "/sbin", "/bin"]
if pkg_manager == "apt-get":
path = ["/usr/local/sbin", "/usr/local/bin"] + path
return {"PATH": ":".join(path), "LC_ALL": "C", "HOME": "/root", "TERM": "dumb"}
def env_traps():
"""Inherited settings that would have leaked into a build. Reported by
`doctor` so the user knows they exist; neutralised by scrubbed_env()."""
found = []
for var in ("LLVM", "LLVM_IAS", "CC", "CXX", "HOSTCC", "HOSTCXX", "LD", "CFLAGS", "LDFLAGS", "KCFLAGS",
"LD_RUN_PATH", "LD_LIBRARY_PATH", "LIBRARY_PATH", "CPATH",
"C_INCLUDE_PATH", "LOCALVERSION", "ARCH", "CROSS_COMPILE",
"PYTHONPATH", "PYTHONHOME", "KCONFIG_CONFIG", "KBUILD_OUTPUT"):
if var in os.environ:
val = os.environ[var]
found.append("%s=%s" % (var, val if len(val) < 60 else val[:57] + "..."))
py = which("python3")
if py and os.path.realpath(py) != os.path.realpath(PLATFORM_PYTHON):
found.append("python3 on PATH is %s, not %s" % (py, PLATFORM_PYTHON))
gcc = which("gcc")
if gcc and os.path.realpath(gcc) != os.path.realpath("/usr/bin/gcc") \
and os.path.exists("/usr/bin/gcc"):
found.append("gcc on PATH is %s, not /usr/bin/gcc" % gcc)
return found
def read_text(path, default=""):
try:
with open(path, "r", errors="replace") as f:
return f.read()
except (OSError, IOError):
return default
def rp(root, path):
return os.path.join(root, path.lstrip("/"))
def human(nbytes):
for unit in ("B", "KB", "MB", "GB", "TB"):
if nbytes < 1024 or unit == "TB":
return "%.1f %s" % (nbytes, unit) if unit != "B" else "%d B" % nbytes
nbytes /= 1024.0
return "%d" % nbytes
def free_bytes(path):
p = path
while p and not os.path.exists(p):
p = os.path.dirname(p)
try:
return shutil.disk_usage(p or "/").free
except OSError:
return 0
# ---------------------------------------------------------------------- versions
def version_key(v):
"""Sortable key. 7.3-rc3 < 7.3 < 7.3.1; anything unparseable sorts first."""
m = re.match(r"^v?(\d+)\.(\d+)(?:\.(\d+))?(?:-rc(\d+))?", v or "")
if not m:
return (0, 0, 0, 0, 0)
a, b, c, rc = m.groups()
return (int(a), int(b), int(c or 0), 0 if rc else 1, int(rc or 0))
def series_of(v):
k = version_key(v)
return "%d.%d" % (k[0], k[1])
# -------------------------------------------------------------------- kernel.org
def http_get(url, timeout=30):
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
except (urllib.error.URLError, OSError) as e:
raise ToolError("could not fetch %s: %s" % (url, e))
def parse_releases(raw):
data = json.loads(raw)
rels = []
for r in data.get("releases", []):
rels.append({
"moniker": r.get("moniker", ""),
"version": r.get("version", ""),
"eol": bool(r.get("iseol")),
"date": (r.get("released") or {}).get("isodate", ""),
"source": r.get("source") or "",
})
latest = (data.get("latest_stable") or {}).get("version", "")
return latest, rels
def get_releases():
return parse_releases(http_get(KORG_RELEASES).decode("utf-8", "replace"))
SUM_LINE = re.compile(r"^([0-9a-f]{64})\s+linux-(\d+\.\d+(?:\.\d+)?)\.tar\.xz\s*$")
def signed_body(text):
"""Only what the PGP signature covers. gpg verifies the clearsigned block
and ignores whatever surrounds it, so a checksum line appended AFTER the
signature would sail through `gpg --verify` and still be parsed -- unless
the parser stops where the signature does."""
m = re.search(r"-----BEGIN PGP SIGNED MESSAGE-----\r?\n(?:[^\r\n]+\r?\n)*\r?\n(.*?)"
r"\r?\n-----BEGIN PGP SIGNATURE-----", text, re.S)
return m.group(1) if m else text
def parse_sha256sums(text):
sums = {}
for line in signed_body(text).splitlines():
m = SUM_LINE.match(line)
if m and m.group(2) not in sums: # first wins; a second entry is suspect
sums[m.group(2)] = m.group(1)
return sums
def get_series_sums(major):
"""One fetch yields every point release in vN.x AND its checksum."""
url = "%s/v%d.x/sha256sums.asc" % (KORG_CDN, major)
text = http_get(url, timeout=60).decode("utf-8", "replace")
return parse_sha256sums(text), text
def resolve_kernel(selector, latest, rels, sums_for):
"""selector: stable | longterm | mainline | X.Y | X.Y.Z | X.Y-rcN
sums_for(major) -> (dict version->sha256, signed text). Injected so the
resolution rules can be tested without a network."""
sel = (selector or "stable").strip().lower()
by_ver = dict((r["version"], r) for r in rels)
ver = None
if sel in ("stable", "latest", "latest-stable"):
ver = latest or next((r["version"] for r in rels if r["moniker"] == "stable"), None)
elif sel in ("longterm", "lts", "latest-longterm"):
ver = next((r["version"] for r in rels
if r["moniker"] == "longterm" and not r["eol"]), None)
elif sel == "mainline":
ver = next((r["version"] for r in rels if r["moniker"] == "mainline"), None)
elif re.match(r"^\d+\.\d+$", sel):
sums, _ = sums_for(int(sel.split(".")[0]))
cands = [v for v in sums if series_of(v) == sel]
if not cands:
raise ToolError("kernel.org has no %s.y releases" % sel)
ver = max(cands, key=version_key)
elif re.match(r"^\d+\.\d+(\.\d+)?(-rc\d+)?$", sel):
ver = sel
else:
raise ToolError("cannot read kernel selector '%s' -- use stable, longterm, "
"mainline, a series (6.12) or a version (6.12.9)" % selector)
if not ver:
raise ToolError("kernel.org's release feed has no '%s' entry right now" % sel)
major = version_key(ver)[0]
if major < 3:
raise ToolError("kernels before 3.0 are not supported")
target = {"version": ver, "sha256": None, "signed_text": None,
"eol": by_ver.get(ver, {}).get("eol", False),
"moniker": by_ver.get(ver, {}).get("moniker", "")}
if "-rc" in ver:
src = by_ver.get(ver, {}).get("source", "")
if not src:
raise ToolError("%s is not the current mainline -rc; kernel.org only "
"publishes a tarball for the newest one" % ver)
target["url"] = src
target["tarball"] = os.path.basename(src)
else:
sums, text = sums_for(major)
if ver not in sums:
raise ToolError("kernel.org has no linux-%s.tar.xz (series %s has %d releases)"
% (ver, series_of(ver),
len([v for v in sums if series_of(v) == series_of(ver)])))
target["sha256"] = sums[ver]
target["signed_text"] = text
target["tarball"] = "linux-%s.tar.xz" % ver
target["url"] = "%s/v%d.x/%s" % (KORG_CDN, major, target["tarball"])
return target
def sha256_file(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def pgp_check(signed_text):
"""(state, detail). state: good | nokey | bad | unavailable."""
gpg = which("gpg") or which("gpg2")
if not gpg:
return "unavailable", "gpg is not installed"
fd, tmp = tempfile.mkstemp(prefix="elkt-sums-", suffix=".asc")
try:
with os.fdopen(fd, "w") as f:
f.write(signed_text)
p = run([gpg, "--batch", "--status-fd", "1", "--verify", tmp], check=False)
finally:
os.unlink(tmp)
o = p.stdout or ""
if "[GNUPG:] BADSIG" in o:
return "bad", "BAD signature on sha256sums.asc"
m = re.search(r"\[GNUPG:\] GOODSIG \S+ (.*)", o)
if m and "[GNUPG:] VALIDSIG" in o:
return "good", m.group(1).strip()
m = re.search(r"\[GNUPG:\] NO_PUBKEY (\S+)", o)
if m:
return "nokey", m.group(1)
return "unavailable", "gpg could not verify the file"
def download(url, dest, sha256=None):
"""To dest.part, hashed as it streams, renamed only once it checks out --
an interrupted download can never be mistaken for a finished one."""
part = dest + ".part"
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
h = hashlib.sha256()
try:
with urllib.request.urlopen(req, timeout=60) as r, open(part, "wb") as f:
total = int(r.headers.get("Content-Length") or 0)
done, last = 0, -1
while True:
chunk = r.read(1 << 20)
if not chunk:
break
f.write(chunk)
h.update(chunk)
done += len(chunk)
pct = int(done * 100 / total) if total else 0
if UI.tty:
sys.stdout.write("\r downloading %3d%% %s / %s "
% (pct, human(done), human(total)))
sys.stdout.flush()
elif pct // 25 != last:
last = pct // 25
info("downloading %3d%%" % pct)
if UI.tty:
sys.stdout.write("\n")
if total and done != total:
# read() returns short, silently, when the peer closes early.
raise OSError("connection closed after %d of %d bytes" % (done, total))
except (urllib.error.URLError, OSError) as e:
if os.path.exists(part):
os.unlink(part)
raise ToolError("download failed: %s: %s" % (url, e))
if sha256 and h.hexdigest() != sha256:
os.unlink(part)
raise ToolError("sha256 MISMATCH for %s\n expected %s\n got %s\n"
"The file has been deleted. Do not retry blindly if this repeats."
% (os.path.basename(dest), sha256, h.hexdigest()))
os.replace(part, dest)
def extract(tarball, tree):
"""Into a scratch directory, then one rename. A half-extracted tree has a
Makefile long before it has drivers/, so its mere existence proves nothing;
the rename is what makes `tree exists` mean `tree is complete`."""
work = os.path.dirname(tree)
tmp = tempfile.mkdtemp(prefix=".extracting-", dir=work)
try:
run(["tar", "-xf", tarball, "-C", tmp])
tops = os.listdir(tmp)
if len(tops) != 1 or not os.path.isfile(os.path.join(tmp, tops[0], "Makefile")):
raise ToolError("%s does not look like a kernel source archive" % tarball)
os.rename(os.path.join(tmp, tops[0]), tree)
finally:
shutil.rmtree(tmp, ignore_errors=True)
def fetch_source(target, workdir, require_pgp=False):
tree = os.path.join(workdir, "linux-" + target["version"])
if os.path.isfile(os.path.join(tree, "Makefile")):
info("source : %s (already extracted)" % tree)
return tree
if os.path.exists(tree):
raise ToolError("%s exists but is not a kernel tree (no Makefile). This tool "
"never leaves one like that; move it away or use another --workdir."
% tree)
os.makedirs(workdir, exist_ok=True)
for stale in glob.glob(os.path.join(workdir, ".extracting-*")):
# Old ones only: a young one is another instance, mid-extract.
if time.time() - os.path.getmtime(stale) > 6 * 3600:
shutil.rmtree(stale, ignore_errors=True)
if target.get("signed_text"):
state, detail = pgp_check(target["signed_text"])
if state == "good":
ok("PGP : sha256sums.asc signed by %s" % detail)
elif state == "bad":
raise ToolError(detail + " -- refusing to use these checksums")
else:
msg = ("sha256sums.asc signature NOT verified (%s)." % (
"key %s is not in your keyring" % detail if state == "nokey" else detail))
if require_pgp:
raise ToolError(msg + "\n Import the kernel.org keys, e.g.\n"
" gpg --locate-keys autosigner@kernel.org "
"torvalds@kernel.org gregkh@kernel.org")
note("PGP : " + msg)
note(" The checksum still proves the download is intact; it")
note(" does not prove who published it. --require-pgp enforces.")
else:
warn("%s is a git snapshot: kernel.org publishes no checksum for it."
% target["tarball"])
if require_pgp:
raise ToolError("--require-pgp cannot be satisfied for -rc snapshots")
tarball = os.path.join(workdir, target["tarball"])
have = os.path.isfile(tarball)
if have and target["sha256"] and sha256_file(tarball) != target["sha256"]:
warn("cached %s fails its checksum -- downloading again" % target["tarball"])
have = False
if not have:
say("Downloading %s" % target["tarball"])
download(target["url"], tarball, target["sha256"])
if target["sha256"]:
ok("sha256 : %s %s" % (target["sha256"][:16] + "...", target["tarball"]))
info("extracting : %s" % tree)
try:
extract(tarball, tree)
except ToolError:
if not target["sha256"] and os.path.exists(tarball):
os.unlink(tarball) # nothing vouches for it; never reuse it
raise
return tree
# --------------------------------------------------------------------- detection
GPU_VENDORS = {0x1002: "amd", 0x10de: "nvidia", 0x8086: "intel"}
NPU_VENDORS = {0x1022: "amd", 0x8086: "intel"}
OTHER_VIDEO = {0x1af4: "virtio", 0x15ad: "vmware", 0x1b36: "qxl", 0x1234: "bochs",
0x1a03: "aspeed-bmc", 0x102b: "matrox-bmc", 0x1414: "hyperv",
0x1013: "cirrus", 0x80ee: "virtualbox"}
def pci_name(vendor, device, root="/"):
"""Best-effort lookup in pci.ids. Purely cosmetic -- decisions use the ids."""
want_v, want_d = "%04x" % vendor, "%04x" % device
for cand in ("usr/share/hwdata/pci.ids", "usr/share/misc/pci.ids",
"usr/share/pci.ids"):
path = rp(root, cand)
if not os.path.isfile(path):
continue
vname, inside = "", False
with open(path, "r", errors="replace") as f:
for line in f:
if line.startswith("#") or not line.strip():
continue
if not line.startswith("\t"):
if inside:
return vname
inside = line.startswith(want_v)
if inside:
vname = line[4:].strip()
elif inside and not line.startswith("\t\t") and line[1:5] == want_d:
return "%s %s" % (vname, line[5:].strip())
if vname:
return vname
return ""
def scan_pci(root="/"):
"""Display controllers (class 03xx) and processing accelerators (1200).
Class and vendor ids from sysfs: no lspci needed, and no matching on the
marketing name, where 'AMD' or 'ATI' can appear in somebody else's string."""
base = rp(root, "sys/bus/pci/devices")
devs = []
try:
names = sorted(os.listdir(base))
except OSError:
return devs
for name in names:
d = os.path.join(base, name)
try:
cls = int(read_text(d + "/class").strip() or "0", 16)
ven = int(read_text(d + "/vendor").strip() or "0", 16)
dev = int(read_text(d + "/device").strip() or "0", 16)
except ValueError:
continue
if cls >> 16 == 0x03:
kind = "display"
elif cls >> 8 == 0x1200:
kind = "accelerator"
else:
continue
drv = ""
if os.path.islink(d + "/driver"):
drv = os.path.basename(os.readlink(d + "/driver"))
devs.append({"addr": name, "class": cls, "vendor": ven, "device": dev,
"kind": kind, "driver": drv,
"name": pci_name(ven, dev, root)})
return devs
def gpu_vendors_present(devs):
found = set()
for d in devs:
table = GPU_VENDORS if d["kind"] == "display" else NPU_VENDORS
if d["vendor"] in table:
found.add(table[d["vendor"]])
return found
def parse_os_release(text):
rel = {}
for line in text.splitlines():
if "=" in line and not line.startswith("#"):
k, v = line.split("=", 1)
rel[k.strip()] = v.strip().strip('"').strip("'")
return rel
PKG_MANAGERS = ("dnf", "yum", "apt-get", "zypper", "pacman")
BUILD_DEPS = {
"dnf": "gcc make flex bison bc openssl openssl-devel elfutils-libelf-devel dwarves "
"ncurses-devel rsync cpio perl python3 xz zstd tar kmod",
"apt-get": "build-essential flex bison bc openssl libssl-dev libelf-dev dwarves "
"libncurses-dev rsync cpio perl python3 xz-utils zstd tar kmod",
"zypper": "gcc make flex bison bc openssl libopenssl-devel libelf-devel dwarves "
"ncurses-devel rsync cpio perl python3 xz zstd tar kmod",
"pacman": "base-devel bc openssl libelf pahole ncurses rsync cpio perl python xz "
"zstd tar kmod",
}
BUILD_DEPS["yum"] = BUILD_DEPS["dnf"]
INSTALL_CMD = {
"dnf": ["dnf", "install", "-y"], "yum": ["yum", "install", "-y"],
"apt-get": ["apt-get", "install", "-y"],
"zypper": ["zypper", "--non-interactive", "install"],
"pacman": ["pacman", "-S", "--needed", "--noconfirm"],
}
def find_pkg_manager():
for pm in PKG_MANAGERS:
if which(pm, path=":".join(SYSTEM_PATH)):
return pm
return ""
def cpu_info(root="/"):
text = read_text(rp(root, "proc/cpuinfo"))
first = text.split("\n\n", 1)[0]
fields = {}
for line in first.splitlines():
if ":" in line:
k, v = line.split(":", 1)
fields[k.strip()] = v.strip()
flags = set((fields.get("flags") or fields.get("Features") or "").split())
try:
n = len(os.sched_getaffinity(0))
except (AttributeError, OSError):
n = os.cpu_count() or 1
return {"vendor": fields.get("vendor_id", ""), "model": fields.get("model name", ""),
"flags": flags, "threads": n, "hypervisor": "hypervisor" in flags}
def mem_total(root="/"):
m = re.search(r"^MemTotal:\s+(\d+) kB", read_text(rp(root, "proc/meminfo")), re.M)
return int(m.group(1)) * 1024 if m else 0
def secure_boot(root="/"):
"""'on' | 'off' | 'n/a' (not an EFI boot). Straight from the EFI variable:
four attribute bytes, then the value. No mokutil needed."""
if not os.path.isdir(rp(root, "sys/firmware/efi")):
return "n/a"
pat = rp(root, "sys/firmware/efi/efivars/SecureBoot-*")
for path in glob.glob(pat):
try:
with open(path, "rb") as f:
raw = f.read()
return "on" if len(raw) >= 5 and raw[4] == 1 else "off"
except OSError:
pass
return "off"
def parse_mounts(text):
out_ = []
for line in text.splitlines():
parts = line.split()
if len(parts) >= 3:
out_.append((parts[0], parts[1].replace("\\040", " "), parts[2]))
return out_
def storage_stack(root="/"):
"""What this machine needs in order to mount its filesystems at boot: the
filesystem types of every /dev-backed mount, and the block layers under
them (LVM, dm-crypt, md). A kernel that lacks one of these builds fine,
installs fine, and drops to an emergency shell."""
mounts = [m for m in parse_mounts(read_text(rp(root, "proc/mounts")))
if m[0].startswith("/dev/")
and not m[1].startswith(("/run/media", "/media", "/mnt"))]
fstypes, layers, rootfs = set(), set(), ""
seen = set()
def walk(dev):
if dev in seen:
return
seen.add(dev)
sysb = rp(root, "sys/class/block/" + dev)
if dev.startswith("dm-"):
uuid = read_text(sysb + "/dm/uuid").strip()
layers.add("lvm" if uuid.startswith("LVM-") else
"crypt" if uuid.startswith("CRYPT-") else "dm")
elif re.match(r"^md\d", dev):
layers.add("md:" + (read_text(sysb + "/md/level").strip() or "?"))
try:
for s in os.listdir(sysb + "/slaves"):
walk(s)
except OSError:
pass
for src, mnt, fstype in mounts:
fstypes.add(fstype)
if mnt == "/":
rootfs = fstype
real = os.path.realpath(src) if root == "/" else src
walk(os.path.basename(real))
return {"rootfs": rootfs, "fstypes": sorted(fstypes), "layers": sorted(layers)}
def firmware_compression(root="/"):
exts = set()
for sub in ("amdgpu", "i915", "nvidia", ""):
d = rp(root, "lib/firmware/" + sub)
try:
for n in os.listdir(d)[:400]:
if n.endswith(".xz"):
exts.add("xz")
elif n.endswith(".zst"):
exts.add("zstd")
except OSError:
pass
return sorted(exts)
# dpcpp is a C++ driver: it belongs to CXX_NAME below.
COMPILER_NAME = re.compile(r"^(gcc|clang)(-\d+(\.\d+)*)?$|^(icx|icc|amdclang|nvc|armclang)$")
# kbuild's HOSTCXX. It compiles the GCC plugins (against the plugin headers of
# the chosen gcc, so it has to MATCH that gcc) and xconfig. `c++` is left out:
# it is g++ under a second name and a second inode.
CXX_NAME = re.compile(r"^(g\+\+|clang\+\+)(-\d+(\.\d+)*)?$|^(icpx|dpcpp|amdclang\+\+)$")
PYTHON_NAME = re.compile(r"^(python3(\.\d+)?|pypy3?)$")
LLVM_TOOLS = ("ld.lld", "llvm-ar", "llvm-nm", "llvm-objcopy", "llvm-objdump",
"llvm-readelf", "llvm-strip")
# Where compilers get installed without anybody adding them to PATH.
TOOL_GLOBS = ("/usr/local/bin", "/opt/*/bin", "/opt/*/*/bin", "~/opt/*/bin", "~/.local/bin",
"/usr/lib/llvm-*/bin", "/usr/lib64/llvm*/bin", "/opt/rh/*/root/usr/bin",
"/opt/intel/oneapi/compiler/*/bin", "/opt/rocm/bin", "/opt/rocm/llvm/bin",
"/opt/AMD/aocc-compiler-*/bin", "/opt/nvidia/hpc_sdk/Linux_*/*/compilers/bin")
TRIPLE_ARCH = {"ppc64le": "powerpc64le", "ppc64": "powerpc64", "armv7l": "arm", "i686": "i"}
def numver(text):
"""(14, 3, 1) out of a version banner. The number after a closing paren
wins -- 'gcc (crosstool-NG 1.26.0) 13.2.0' -- else the first dotted one,
else the first integer ('kmod version 31')."""
m = re.search(r"\)\s+(\d+(?:\.\d+)+)", text) or re.search(r"(\d+(?:\.\d+)+)", text) \
or re.search(r"(\d+)", text)
return tuple(int(x) for x in m.group(1).split(".")) if m else ()
def ver_ge(have, need):
n = max(len(have), len(need), 3)
pad = lambda v: tuple(v) + (0,) * (n - len(v))
return pad(have) >= pad(need)
def vstr(v):
return ".".join(str(x) for x in v) if v else "?"
def tool_search_dirs(root="/", home=None, path_env=None):
"""/usr/bin first, so a system tool is met under its plain name before any
alias of it; then the caller's PATH, in their order; then TOOL_GLOBS."""
home = os.path.expanduser("~") if home is None else home
path_env = os.environ.get("PATH", "") if path_env is None else path_env
dirs = [rp(root, "usr/bin")] + [d for d in path_env.split(":") if d]
for pat in TOOL_GLOBS:
pat = home + pat[1:] if pat.startswith("~") else rp(root, pat)
dirs.extend(sorted(glob.glob(pat)))
seen, res = set(), []
for d in dirs:
real = os.path.realpath(d)
if real not in seen and os.path.isdir(d):
seen.add(real)
res.append(d)
return res
def scan_tool_dirs(dirs):
"""-> (C compilers, pythons, C++ compilers): executable FILES whose names
look the part, one per real file PER KIND -- clang++ is a symlink to the
very binary clang is. PATH entries like ~/opt/gcc-17/lib hold a DIRECTORY
named gcc; the isfile() is not decoration."""
found = {"cc": [], "py": [], "cxx": []}
seen = {"cc": set(), "py": set(), "cxx": set()}
for d in dirs:
try:
names = sorted(os.listdir(d), key=lambda n: (len(n), n)) # 'clang' before 'clang-21'
except OSError:
continue
for name in names:
key = "cc" if COMPILER_NAME.match(name) else "cxx" if CXX_NAME.match(name) \
else "py" if PYTHON_NAME.match(name) else None
if not key:
continue
path = os.path.join(d, name)
real = os.path.realpath(path)
if real in seen[key] or not os.path.isfile(path) or not os.access(path, os.X_OK):
continue
seen[key].add(real)
found[key].append(path)
return found["cc"], found["py"], found["cxx"]
def compiler_info(path):
"""What a compiler says it is -- and whether kbuild can use it.
kbuild supports GCC and Clang, nothing else: ICC support was REMOVED from
Linux in 6.2. Intel's icx/DPC++ is LLVM-based and defines __clang__, so
kbuild's own probe would file it as 'Clang 22' and carry on; its banner
says neither 'gcc' nor 'clang', so a name test would file it as gcc. Hence
Intel is looked for FIRST, and vendor forks of clang are marked as such."""
try:
p = run([path, "--version"], check=False, timeout=20)
except ToolError:
return None
lines = [l.strip() for l in (p.stdout or "").splitlines()
if l.strip() and "warning:" not in l]
if p.returncode != 0 or not lines:
return None
line, real = lines[0], os.path.realpath(path)
inst = re.search(r"^InstalledDir:\s*(\S+)", p.stdout or "", re.M)
low = line.lower()
if "intel" in low or "(ifx)" in low or "(icc)" in low:
kind = "intel"
elif "clang" in low:
kind = "clang"
elif "gcc" in low or "free software foundation" in " ".join(lines).lower():
kind = "gcc"
else:
kind = "other"
usable, why = "yes", ""
if real.startswith(("/usr/bin/", "/usr/lib", "/usr/libexec/")):
origin = "system"
elif "/opt/rh/" in real or "gcc-toolset" in real or "devtoolset" in real:
origin = "Red Hat gcc-toolset (vendor-supported)"
elif "/rocm" in real or os.path.basename(path) == "amdclang":
origin, usable = "ROCm -- AMD's fork of LLVM, built for HIP", "fork"
why = "a vendor fork of clang: nobody tests kernels with it"
elif "/aocc" in real.lower():
origin, usable = "AOCC -- AMD's fork of LLVM", "fork"
why = "a vendor fork of clang: nobody tests kernels with it"
elif "/opt/intel" in real:
origin = "Intel oneAPI"
elif "/opt/nvidia" in real:
origin = "NVIDIA HPC SDK"
else:
origin = "local build"
if kind == "intel":
usable = "no"
why = ("kbuild supports GCC and Clang only: ICC support was removed in Linux 6.2, "
"and icx/DPC++ -- LLVM underneath -- is not a supported kernel compiler")
elif kind == "other":
usable, why = "no", "neither GCC nor Clang"
triple = ""
if usable != "no":
try:
triple = (run([path, "-dumpmachine"], check=False, timeout=20).stdout or "").strip()
except ToolError:
pass
arch = os.uname().machine
if triple and not triple.startswith(TRIPLE_ARCH.get(arch, arch)):
usable, why = "no", "targets %s, not this machine (%s)" % (triple, arch)
return {"path": path, "real": real, "kind": kind, "version": line, "numver": numver(line),
"system": origin == "system", "origin": origin, "usable": usable, "why": why,
"triple": triple, "installed": inst.group(1) if inst else "",
"experimental": bool(re.search(r"experimental|prerelease|\dgit\b", line))}
def compiler_works(path, extra_path=None):
"""Can it turn a C file into an executable IN THE SCRUBBED ENVIRONMENT the
build will run in? A private compiler that needs LD_LIBRARY_PATH to find
its own libisl works in your shell and dies at the first object here."""
try:
p = run([path, "-x", "c", "-", "-o", os.devnull], check=False, timeout=60,
stdin_data="int main(void) { return 0; }\n", env=scrubbed_env(extra_path))
except ToolError as e:
return False, str(e).splitlines()[0]
if p.returncode == 0:
return True, ""
return False, ((p.stdout or "").strip().splitlines() or ["exit %d" % p.returncode])[-1][:120]
def llvm_companions(clang_path):
"""-> (LLVM= argument, extra PATH dir or None), or None when the set beside
this clang is incomplete. kbuild takes LLVM=1 (plain names on PATH),
LLVM=/dir/ (a prefix) or LLVM=-18 (a suffix: Debian's clang-18, ld.lld-18)."""
d, name = os.path.split(clang_path)
m = re.match(r"^clang(-\d+(?:\.\d+)*)$", name)
suffix = m.group(1) if m else ""
if not all(os.access(os.path.join(d, t + suffix), os.X_OK) for t in LLVM_TOOLS):
return None
on_path = os.path.realpath(d) in [os.path.realpath(x) for x in SYSTEM_PATH[:4]]
if suffix:
return "LLVM=" + suffix, None if on_path else d
if on_path:
return "LLVM=1", None
return "LLVM=" + d + "/", None
def own_binutils(cc_path):
"""The directory, when a compiler ships its own ld AND as beside it (every
gcc-toolset does) -- they were built and tested together."""
d = os.path.dirname(cc_path)
both = all(os.access(os.path.join(d, t), os.X_OK) for t in ("ld", "as"))
return d if both and os.path.realpath(d) != os.path.realpath("/usr/bin") else ""
def _collect(cands, plain):
"""One entry per COMPILER, not per name and not per banner. amdclang is
ROCm's clang and dpcpp is icx: same banner, same InstalledDir -- one entry,
under the plain name. Two installs built from one commit have the same
banner too, and are two compilers: InstalledDir tells them apart. (Merging
on the banner alone hid exactly those.)"""
cands = sorted(cands, key=lambda c: os.path.basename(c) not in plain)
found, keys = [], set()
for c in cands:
ci = compiler_info(c)
if not ci:
continue
key = (ci["version"], ci["installed"]) if ci["installed"] else ("", ci["real"])
if key not in keys:
keys.add(key)
found.append(ci)
rank = {"yes": 0, "fork": 1, "no": 2}
found.sort(key=lambda t: (rank[t["usable"]], not t["system"], t["kind"] != "gcc",
tuple(-x for x in t["numver"])))
return found
def find_toolchains(dirs=None):
dirs = tool_search_dirs() if dirs is None else dirs
return _collect(scan_tool_dirs(dirs)[0], ("gcc", "clang", "icx"))
def find_cxx(dirs=None):
dirs = tool_search_dirs() if dirs is None else dirs
return _collect(scan_tool_dirs(dirs)[2], ("g++", "clang++", "icpx"))
def cxx_partner(cc_path):
"""The C++ driver installed beside a C compiler: gcc-13 -> g++-13."""
d, name = os.path.split(cc_path)
m = re.match(r"^(gcc|clang)(-\d+(?:\.\d+)*)?$", name)
if not m:
return ""
cand = os.path.join(d, ("g++" if m.group(1) == "gcc" else "clang++") + (m.group(2) or ""))
return cand if os.access(cand, os.X_OK) else ""
def find_binutils(dirs=None):
"""Every GNU binutils set: a directory holding both ld and as."""
dirs = tool_search_dirs() if dirs is None else dirs
sets, inodes = [], set()
for d in sorted(dirs, key=lambda x: (os.path.realpath(x) != os.path.realpath("/usr/bin"), len(x))):
ld = os.path.join(d, "ld")
if not all(os.access(os.path.join(d, t), os.X_OK) for t in ("ld", "as")):
continue
st = os.stat(ld)
if (st.st_dev, st.st_ino) in inodes:
continue # <prefix>/<triple>/bin/ld is a hard link to <prefix>/bin/ld
inodes.add((st.st_dev, st.st_ino))
try:
line = ((run([ld, "-v"], check=False, timeout=20).stdout or "").splitlines() or [""])[0]
except ToolError:
continue
# STARTS with: lld's banner is "LLD 18.1.3 (compatible with GNU linkers)",
# and gold's is "GNU gold" -- an `ld` that is really either is not bfd.
if not line.startswith("GNU ld"):
continue
if any(b["version"] == line.strip() and os.path.commonpath([b["dir"], d]) not in ("/", "")
and os.path.dirname(b["dir"]) in d for b in sets):
continue # ...or a copy of it, under the same prefix
sets.append({"dir": d, "version": line.strip(), "numver": numver(line),
"system": os.path.realpath(d) == os.path.realpath("/usr/bin")})
return sets
def find_pythons(dirs=None):
dirs = tool_search_dirs() if dirs is None else dirs
res = []
# (index 1 of scan_tool_dirs: the pythons)
probe = ("import sys, platform; print(platform.python_implementation(), "
"'.'.join(str(x) for x in sys.version_info[:3]))")
for path in scan_tool_dirs(dirs)[1]:
try:
o = (run([path, "-c", probe], check=False, timeout=20).stdout or "").strip()
except ToolError:
continue
m = re.match(r"^(\w+) (\d+\.\d+\.\d+)$", o.splitlines()[-1] if o else "")
if not m or int(m.group(2).split(".")[0]) < 3:
continue
res.append({"path": path, "impl": m.group(1), "version": m.group(2),
"numver": numver(m.group(2)),
"platform": os.path.realpath(path) == os.path.realpath(PLATFORM_PYTHON)})
res.sort(key=lambda y: (not y["platform"], y["impl"] != "CPython",
tuple(-x for x in y["numver"])))
return res
MISC_TOOLS = (("make", "--version"), ("ld.bfd", "-v"), ("ld.lld", "-v"), ("ld.gold", "-v"),
("mold", "-v"), ("ccache", "--version"), ("pahole", "--version"),
("rustc", "--version"), ("bindgen", "--version"))
NOT_FOR_KERNELS = {"ld.gold": "the kernel refuses gold outright",
"mold": "kbuild does not support mold"}
def find_misc_tools():
res = {}
for name, flag in MISC_TOOLS:
path = which(name, path=":".join(SYSTEM_PATH))
if path:
try:
o = run([path, flag], check=False, timeout=20).stdout or ""
except ToolError:
o = ""
res[name] = {"path": path, "version": (o.strip().splitlines() or [""])[0][:70]}
return res
DKMS_LINE = re.compile(r"^([^/,\s]+)[/,]\s*([^,\s]+),\s*([^,\s]+),\s*[^:]+:\s*(\S+)")
def parse_dkms_status(text):
mods = {}
for line in text.splitlines():
m = DKMS_LINE.match(line.strip())
if m:
name, ver, kernel, state = m.groups()
mods.setdefault((name, ver), []).append((kernel, state))
return mods
def parse_dkms_conf(text):
lim = {}
for key in ("BUILD_EXCLUSIVE_KERNEL_MIN", "BUILD_EXCLUSIVE_KERNEL_MAX",
"BUILD_EXCLUSIVE_KERNEL"):
m = re.search(r"^%s=[\"']?([^\"'\n]+)[\"']?\s*$" % key, text, re.M)
if m:
lim[key] = m.group(1)
return lim
def dkms_will_build(limits, release):
"""Mirror dkms's own rule: skip when VER(kernel) > MAX or < MIN, or when
the BUILD_EXCLUSIVE_KERNEL regex does not match. Note 6.18.52 > '6.18'."""
k = version_key(release)[:3]
if "BUILD_EXCLUSIVE_KERNEL_MAX" in limits:
if k > version_key(limits["BUILD_EXCLUSIVE_KERNEL_MAX"])[:3]:
return False, "capped at %s" % limits["BUILD_EXCLUSIVE_KERNEL_MAX"]
if "BUILD_EXCLUSIVE_KERNEL_MIN" in limits:
if k < version_key(limits["BUILD_EXCLUSIVE_KERNEL_MIN"])[:3]:
return False, "needs at least %s" % limits["BUILD_EXCLUSIVE_KERNEL_MIN"]
if "BUILD_EXCLUSIVE_KERNEL" in limits:
try:
if not re.search(limits["BUILD_EXCLUSIVE_KERNEL"], release):
return False, "BUILD_EXCLUSIVE_KERNEL does not match"
except re.error:
pass
return True, ""
def dkms_modules(root="/"):
"""From /var/lib/dkms itself -- <module>/<version>/<kernel>/, with a
kernel-<k>-<arch> symlink per INSTALLED build. `dkms status` says the same
in three seconds of bash; this is every command's start-up path."""
base = rp(root, "var/lib/dkms")
try:
names = sorted(n for n in os.listdir(base) if os.path.isdir(os.path.join(base, n)))
except OSError:
names = None
if names is None:
if not which("dkms", path=":".join(SYSTEM_PATH)):
return []
p = run(["dkms", "status"], check=False, env=scrubbed_env(), timeout=60)
return [{"name": n, "version": v, "kernels": k,
"limits": parse_dkms_conf(read_text(rp(root, "usr/src/%s-%s/dkms.conf" % (n, v))))}
for (n, v), k in sorted(parse_dkms_status(p.stdout or "").items())]
mods = []
for name in names:
mdir = os.path.join(base, name)
installed = set()
for entry in os.listdir(mdir):
full = os.path.join(mdir, entry)
if entry.startswith("kernel-") and os.path.islink(full):
parts = os.readlink(full).split("/")
if len(parts) >= 2:
installed.add((parts[0], parts[1]))
for ver in sorted(os.listdir(mdir)):
vdir = os.path.join(mdir, ver)
if os.path.islink(vdir) or not os.path.isdir(vdir) or ver == "original_module":
continue
kernels = [(k, "installed" if (ver, k) in installed else "built")
for k in sorted(os.listdir(vdir))
if k not in ("source", "build") and os.path.isdir(os.path.join(vdir, k))
and not os.path.islink(os.path.join(vdir, k))]
conf = read_text(os.path.join(vdir, "source", "dkms.conf")) or \
read_text(rp(root, "usr/src/%s-%s/dkms.conf" % (name, ver)))
mods.append({"name": name, "version": ver, "kernels": kernels,
"limits": parse_dkms_conf(conf)})
return mods
def detect_machine():
osr = parse_os_release(read_text("/etc/os-release"))
devs = scan_pci()
virt = ""
if which("systemd-detect-virt"):
virt = (run(["systemd-detect-virt"], check=False).stdout or "").strip()
cpu = cpu_info()
if not virt or virt == "none":
virt = "yes (unidentified)" if cpu["hypervisor"] else "none"
tool_dirs = tool_search_dirs()
initramfs = next((t for t in ("dracut", "update-initramfs", "mkinitcpio")
if which(t, path=":".join(SYSTEM_PATH))), "")
return {
"os": osr.get("PRETTY_NAME", osr.get("ID", "unknown")),
"os_id": osr.get("ID", ""), "os_like": osr.get("ID_LIKE", ""),
"arch": os.uname().machine, "running": os.uname().release,
"pkg_manager": find_pkg_manager(),
"cpu": {"vendor": cpu["vendor"], "model": cpu["model"],
"threads": cpu["threads"],
"simd": [f for f in ("avx512f", "avx2", "avx", "sse4_2") if f in cpu["flags"]][:1]},
"memory": mem_total(), "virt": virt,
"pci": devs, "gpu_vendors": sorted(gpu_vendors_present(devs)),
"storage": storage_stack(), "firmware_compression": firmware_compression(),
"secure_boot": secure_boot(),
"boot": {"grubby": bool(which("grubby", path=":".join(SYSTEM_PATH))),
"bls": os.path.isdir("/boot/loader/entries"), "initramfs": initramfs},
"toolchains": find_toolchains(tool_dirs), "cxx": find_cxx(tool_dirs),
"binutils": find_binutils(tool_dirs),
"pythons": find_pythons(tool_dirs), "dkms": dkms_modules(),
"loaded_modules": len(read_text("/proc/modules").splitlines()),
}
# ----------------------------------------------------------------------- .config
class DotConfig(object):
"""A kernel .config, edited in memory and written atomically.
set() REMOVES the symbol's existing line and APPENDS the new one. Kconfig
resolves a choice group by 'the last =y read wins', and scripts/config
edits in place -- so enabling DEBUG_INFO_NONE there rewrote a line sitting
ABOVE the distro's DWARF selection and silently lost. Appending makes the
value we set the last one read, for every choice, without special cases."""
LINE = re.compile(r"^(?:CONFIG_([A-Za-z0-9_]+)=(.*)|# CONFIG_([A-Za-z0-9_]+) is not set)\s*$")
def __init__(self, path):
self.path = path
self.lines = read_text(path).splitlines()
def _sym(self, line):
m = self.LINE.match(line)
if not m:
return None, None
if m.group(1):
return m.group(1), m.group(2)
return m.group(3), "n"
def get(self, sym):
"""'y' | 'm' | 'n' | the unquoted value | None when absent entirely."""
val = None
for line in self.lines:
s, v = self._sym(line)
if s == sym:
val = v
if val is not None and len(val) >= 2 and val[0] == '"' and val[-1] == '"':
val = val[1:-1].replace('\\"', '"')
return val
def set(self, sym, rendered):
"""rendered: 'y' 'm' 'n', or the literal right-hand side."""
self.lines = [l for l in self.lines if self._sym(l)[0] != sym]
if rendered == "n":
self.lines.append("# CONFIG_%s is not set" % sym)
else:
self.lines.append("CONFIG_%s=%s" % (sym, rendered))
def count(self, value):
return sum(1 for l in self.lines if self._sym(l)[1] == value)
def valid(self):
return any(self._sym(l)[0] for l in self.lines)
def write(self):
tmp = "%s.tmp.%d" % (self.path, os.getpid())
with open(tmp, "w") as f:
f.write("\n".join(self.lines) + "\n")
os.replace(tmp, self.path)
SRCARCH = {"x86_64": "x86", "i686": "x86", "i386": "x86", "aarch64": "arm64",
"ppc64le": "powerpc", "ppc64": "powerpc", "s390x": "s390",
"riscv64": "riscv", "armv7l": "arm", "loongarch64": "loongarch"}
KCONFIG_DEF = re.compile(r"^\s*(?:menu)?config\s+([A-Za-z0-9_]+)\s*$", re.M)
def kconfig_symbols(tree, machine_arch):
"""Every symbol THIS tree defines for THIS architecture. It is what lets
verify tell 'not in this kernel version' (DRM_XE before 6.8 -- a note) from
'present, requested, and dropped' (a dependency is off -- a failure)."""
arch = SRCARCH.get(machine_arch, machine_arch)
syms = set()
for dirpath, dirnames, filenames in os.walk(tree):
rel = os.path.relpath(dirpath, tree)
if rel == "arch":
dirnames[:] = [d for d in dirnames if d == arch]
elif rel == ".":
dirnames[:] = [d for d in dirnames
if d not in ("Documentation", "tools", "samples", ".git")]
for fn in filenames:
if fn.startswith("Kconfig"):
syms.update(KCONFIG_DEF.findall(read_text(os.path.join(dirpath, fn))))
return syms
class Req(object):
"""One thing asked of the config.
want : 'y' 'm' 'n', 'm|y' (at least a module), ('str', s) or ('val', n)
level: required -> unmet is fatal | wanted -> a warning | optional -> a note"""
def __init__(self, sym, want, level, why=""):
self.sym, self.want, self.level, self.why = sym, want, level, why
def rendered(self, current, attempt=1):
if self.want == "m|y":
# 'm' first; a bool symbol (HYPERV from 6.18 on) rejects it, and the
# second pass asks for 'y' instead.
return current if current in ("m", "y") else ("m" if attempt == 1 else "y")
if isinstance(self.want, tuple):
return '"%s"' % self.want[1] if self.want[0] == "str" else str(self.want[1])
return self.want
def met(self, got):
if self.want == "n":
return got in (None, "n")
if self.want == "m|y":
return got in ("m", "y")
if isinstance(self.want, tuple):
# Asked to be EMPTY (a path to a file the tarball does not ship):
# absent -- its parent option is off -- is every bit as good.
if self.want[1] == "" and got is None:
return True
return got == str(self.want[1])
return got == self.want
def shown(self):
return self.want[1] if isinstance(self.want, tuple) else self.want
FS_SYMBOLS = {"xfs": "XFS_FS", "ext4": "EXT4_FS", "ext3": "EXT4_FS", "ext2": "EXT4_FS",
"btrfs": "BTRFS_FS", "vfat": "VFAT_FS", "f2fs": "F2FS_FS",
"bcachefs": "BCACHEFS_FS", "exfat": "EXFAT_FS", "jfs": "JFS_FS",
"nilfs2": "NILFS2_FS", "erofs": "EROFS_FS", "ntfs3": "NTFS3_FS"}
MD_SYMBOLS = {"raid0": "MD_RAID0", "raid1": "MD_RAID1", "raid10": "MD_RAID10",
"raid4": "MD_RAID456", "raid5": "MD_RAID456", "raid6": "MD_RAID456"}
# Kept alive under --modules trimmed. localmodconfig keeps what is LOADED NOW;
# none of these is loaded on a healthy bare-metal box, and each is what you
# reach for on the day it is not healthy: a USB stick, a rescue ISO, a VM.
RESCUE_SYMBOLS = ("USB_STORAGE", "USB_UAS", "BLK_DEV_LOOP", "ISO9660_FS", "SQUASHFS",
"VFAT_FS", "NLS_CODEPAGE_437", "NLS_ISO8859_1", "NLS_UTF8", "EXT4_FS",
"XFS_FS", "BLK_DEV_NVME", "SATA_AHCI", "BLK_DEV_DM", "VIRTIO_PCI",
"VIRTIO_BLK", "VIRTIO_NET", "SCSI_VIRTIO", "HID_GENERIC", "USB_HID")
PREEMPT_CHOICE = {"none": "PREEMPT_NONE", "voluntary": "PREEMPT_VOLUNTARY",
"full": "PREEMPT", "lazy": "PREEMPT_LAZY"}
HZ_CHOICE = ("100", "250", "300", "1000")
def resolve_gpu(selection, detected):
"""-> (set of vendors, explanation)."""
sel = (selection or "auto").strip().lower()
if sel == "auto":
if detected:
return set(detected), "detected"
return {"amd", "nvidia", "intel"}, \
"none detected (headless, VM or unknown) -- all vendors, so it boots anywhere"
if sel == "all":
return {"amd", "nvidia", "intel"}, "all, as asked"
vendors = set()
for v in sel.split(","):
v = v.strip()
if v in ("amd", "intel"):
vendors.add(v)
elif v in ("nvidia", "nv"):
vendors.add("nvidia")
elif v not in ("vm", "none", ""):
raise ToolError("unknown GPU vendor '%s' (use: auto all none amd nvidia intel)" % v)
return vendors, "as asked"
def build_requests(opts, vendors, machine):
R = []
def add(sym, want, level, why=""):
R.append(Req(sym, want, level, why))
# GPU-compute plumbing. DEPENDENCIES, not preferences: HSA_AMD_SVM needs
# DEVICE_PRIVATE, HSA_AMD_P2P needs PCI_P2PDMA, both need ZONE_DEVICE, and
# ZONE_DEVICE needs memory hotplug -- which a distro config has and
# `make defconfig` does not, so from that base the whole chain used to
# vanish without a line in .config to show for it.
add("MEMORY_HOTPLUG", "y", "wanted", "ZONE_DEVICE depends on it")
add("MEMORY_HOTREMOVE", "y", "wanted", "ZONE_DEVICE depends on it")
add("ZONE_DEVICE", "y", "required", "device memory; under HMM, P2P, UVM")
add("DEVICE_PRIVATE", "y", "required", "HSA_AMD_SVM / nouveau SVM depend on it")
add("PCI_P2PDMA", "y", "required", "GPU peer-to-peer over PCIe")
add("DRM_ACCEL", "y", "wanted", "the subsystem both NPUs live under")
# Out-of-tree modules -- DKMS -- must be buildable and loadable.
add("MODULES", "y", "required", "no modules, no DKMS")
add("MODULE_UNLOAD", "y", "wanted")
add("MODULE_SIG_FORCE", "n", "required", "or no unsigned DKMS module will load")
# A console before any GPU driver binds, and the VM / BMC adapters: a few
# hundred KB that separate 'boots on my machine' from 'boots anywhere'.
# localmodconfig switches DRM itself off on a headless box (no DRM module
# loaded), and every driver below then fails for want of its parent.
add("DRM", "m|y", "required", "parent of every graphics driver")
add("SYSFB_SIMPLEFB", "y", "wanted")
add("DRM_SIMPLEDRM", "m", "required", "EFI/VESA framebuffer -- the rescue console")
add("DRM_FBDEV_EMULATION", "y", "wanted")
add("FRAMEBUFFER_CONSOLE", "y", "wanted")
add("DRM_VIRTIO_GPU", "m", "required", "QEMU/KVM")
for sym in ("DRM_QXL", "DRM_BOCHS", "DRM_AST", "DRM_MGAG200", "DRM_UDL"):
add(sym, "m", "wanted")
add("HYPERV", "m|y", "optional", "DRM_HYPERV depends on it")
for sym in ("DRM_VMWGFX", "DRM_HYPERV", "DRM_EFIDRM", "DRM_VESADRM"):
add(sym, "m", "optional")
if vendors:
# Firmware arrives compressed on every current distro (.xz on EL and
# Fedora, .zst on Ubuntu and Arch). Without the matching decompressor
# the driver loads, asks for firmware, and gets nothing.
add("FW_LOADER", "y", "required")
add("FW_LOADER_COMPRESS", "y", "required", "distro firmware is compressed")
have = machine.get("firmware_compression", [])
add("FW_LOADER_COMPRESS_XZ", "y", "required" if "xz" in have else "wanted")
add("FW_LOADER_COMPRESS_ZSTD", "y", "required" if "zstd" in have else "wanted")
if "amd" in vendors:
add("DRM_AMDGPU", "m", "required", "MODULE, never built-in -- see the file header")
add("DRM_AMD_DC", "y", "required", "atomic KMS, DP/HDMI audio")
add("DRM_AMDGPU_USERPTR", "y", "required", "ROCm, GL/Vulkan interop")
add("DRM_AMDGPU_SI", "y", "wanted")
add("DRM_AMDGPU_CIK", "y", "wanted")
add("HSA_AMD", "y", "required", "KFD: /dev/kfd, ROCm's way in")
add("HSA_AMD_SVM", "y", "required", "HIP managed memory")
# Up to 6.x HSA_AMD_P2P also depends on DMABUF_MOVE_NOTIFY. 7.x dropped
# the symbol, so a 7.x config carried back to an LTS tree lacks it and
# P2P silently disappears. On 7.x this is a 'not in this version' note.
add("DMABUF_MOVE_NOTIFY", "y", "optional", "HSA_AMD_P2P depends on it before 7.x")
add("HSA_AMD_P2P", "y", "required", "GPU-to-GPU transfers")
add("AMD_IOMMU", "y", "optional", "x86 only; DRM_ACCEL_AMDXDNA needs it")
add("DRM_ACCEL_AMDXDNA", "m", "optional", "Ryzen AI NPU; 6.14+")
if "nvidia" in vendors:
# A module even if NVIDIA's own driver is the plan: that stack needs
# nouveau absent at load time, and only a module can be blacklisted.
add("DRM_NOUVEAU", "m", "required")
if "intel" in vendors:
add("DRM_I915", "m", "required")
add("DRM_XE", "m", "optional", "6.8+")
add("DRM_ACCEL_IVPU", "m", "optional", "Meteor Lake+ NPU")
# What this machine needs to mount its own disks.
st = machine.get("storage", {})
for fs in st.get("fstypes", []):
if fs in FS_SYMBOLS:
add(FS_SYMBOLS[fs], "m|y", "required", "a mounted filesystem here is %s" % fs)
if fs == "vfat":
for nls in ("NLS_CODEPAGE_437", "NLS_ISO8859_1", "NLS_ASCII", "NLS_UTF8"):
add(nls, "m|y", "wanted", "vfat mount options")
for layer in st.get("layers", []):
if layer in ("lvm", "dm", "crypt"):
add("BLK_DEV_DM", "m|y", "required", "storage here sits on device-mapper")
if layer == "crypt":
add("DM_CRYPT", "m|y", "required", "storage here is dm-crypt")
if layer.startswith("md:"):
add("BLK_DEV_MD", "m|y", "required", "storage here is md raid")
if layer[3:] in MD_SYMBOLS:
add(MD_SYMBOLS[layer[3:]], "m|y", "required", layer)
if opts["modules"] == "trimmed":
for sym in RESCUE_SYMBOLS + tuple(opts.get("keep_symbols", ())):
add(sym, "m|y", "wanted", "rescue set, kept through the trim")
# Things that break a distro config carried to an upstream tree: signing
# keys, revocation lists and the SBAT file all live in the distro's source
# package, and each is a hard stop late in the build. WERROR is right for
# the one compiler a distro tests and wrong for every other pairing.
add("SYSTEM_TRUSTED_KEYS", ("str", ""), "required")
add("SYSTEM_REVOCATION_KEYS", ("str", ""), "required")
add("EFI_SBAT_FILE", ("str", ""), "wanted")
add("WERROR", "n", "required")
add("FRAME_WARN", ("val", opts["frame_warn"]), "wanted",
"amdgpu's DML files build ~2.4K frames")
# The running kernel should carry its own config. Its absence is what made
# the old script lose its base config on the second use.
add("IKCONFIG", "y", "wanted")
add("IKCONFIG_PROC", "y", "wanted", "/proc/config.gz")
if opts.get("secure_boot", "on") == "on":
# shim boots a SIGNED PE image, so the kernel must BE one (EFI_STUB). Under
# lockdown only signed modules load: the in-tree ones are signed at
# modules_install with the tree's own throw-away key, whose certificate is
# compiled in (what distros do); DKMS signs with the machine's MOK key,
# which the kernel trusts only with the MACHINE keyring. MODULE_SIG_FORCE
# stays off -- lockdown enforces when Secure Boot is on, and with it off an
# unsigned out-of-tree module must still load.
add("EFI", "y", "required", "Secure Boot is a UEFI feature")
add("EFI_STUB", "y", "required", "shim verifies and boots a PE image")
add("MODULE_SIG", "y", "required", "lockdown loads signed modules only")
add("MODULE_SIG_ALL", "y", "required", "sign every module at modules_install")
add("MODULE_SIG_SHA512", "y", "wanted")
add("MODULE_SIG_KEY", ("str", "certs/signing_key.pem"), "wanted",
"kbuild generates this key per tree; a distro path here is a missing file")
add("SECURITY_LOCKDOWN_LSM", "y", "wanted")
add("SECURITY_LOCKDOWN_LSM_EARLY", "y", "wanted")
add("LOCK_DOWN_IN_EFI_SECURE_BOOT", "y", "optional", "a distro patch, not upstream")
add("SECONDARY_TRUSTED_KEYRING", "y", "wanted")
add("INTEGRITY_PLATFORM_KEYRING", "y", "wanted", "firmware db keys")
add("INTEGRITY_MACHINE_KEYRING", "y", "wanted", "MOK keys: DKMS-signed modules load")
add("LOAD_UEFI_KEYS", "y", "wanted")
add("KEXEC_SIG", "y", "optional")
if opts["debug_info"] == "off":
for sym in ("DEBUG_INFO_BTF", "DEBUG_INFO_BTF_MODULES",
"DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT", "DEBUG_INFO_DWARF4",
"DEBUG_INFO_DWARF5"):
add(sym, "n", "optional")
add("DEBUG_INFO_NONE", "y", "required", "else 10x module size, far slower build")
elif opts["debug_info"] == "btf":
# BTF is made FROM DWARF by pahole, so DWARF has to exist at build time;
# modules are stripped of it at install (INSTALL_MOD_STRIP keeps .BTF).
# Without BTF a kernel with BPF_LSM logs 'bpf-restrict-fs: Failed to
# load BPF object' from systemd on every boot, and bpftrace and every
# CO-RE tool (Cilium, Falco, Tetragon agents) cannot run.
for sym in ("DEBUG_INFO_NONE", "DEBUG_INFO_REDUCED", "DEBUG_INFO_SPLIT",
"DEBUG_INFO_DWARF4", "DEBUG_INFO_DWARF5"):
add(sym, "n", "optional")
add("DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT", "y", "required", "BTF is generated from DWARF")
add("DEBUG_INFO_BTF", "y", "required", "systemd BPF filters, bpftrace, CO-RE tools")
add("DEBUG_INFO_BTF_MODULES", "y", "wanted")
if opts["preempt"] != "keep":
for key, sym in sorted(PREEMPT_CHOICE.items()):
if key != opts["preempt"]:
add(sym, "n", "optional")
add(PREEMPT_CHOICE[opts["preempt"]], "y", "wanted", "preemption model")
if opts["hz"] != "keep":
for hz in HZ_CHOICE:
if hz != opts["hz"]:
add("HZ_" + hz, "n", "optional")
add("HZ_" + opts["hz"], "y", "wanted", "timer frequency")
if opts["cpu"] == "native":
add("X86_NATIVE_CPU", "y", "wanted", "-march=native; 6.16+, x86 only")
else:
add("X86_NATIVE_CPU", "n", "optional")
add("LOCALVERSION", ("str", opts["localversion"]), "required")
add("LOCALVERSION_AUTO", "n", "required")
return R
def verify_requests(cfg, reqs, symbols):
"""-> list of (req, got, status): ok | fail | warn | note | absent."""
res = []
for r in reqs:
got = cfg.get(r.sym)
if r.met(got):
res.append((r, got, "ok"))
elif r.sym not in symbols:
res.append((r, got, "absent"))
else:
res.append((r, got, {"required": "fail", "wanted": "warn"}.get(r.level, "note")))
return res
def config_series(path):
"""'6.12' from the '# Linux/x86 6.12.0 Kernel Configuration' header, or ''."""
try:
opener = gzip.open if path.endswith(".gz") else open
with opener(path, "rt", errors="replace") as f:
head = "".join(f.readline() for _ in range(6))
except (OSError, IOError, EOFError):
return ""
m = re.search(r"^# Linux/\S+ (\d+\.\d+)", head, re.M)
return m.group(1) if m else ""
def richest(paths):
"""The candidate with the most modules. /boot also holds configs THIS tool
installed for trimmed kernels; 'newest' would pick one of those."""
return max(paths, key=lambda p: (DotConfig(p).count("m"),
version_key(os.path.basename(p)[7:])))
def find_base_config(tree, mode, running=None, root="/", target_series="",
skip_own=False, want_full=False):
"""-> (kind, path, label). kind: file | gz | defconfig.
The chain is long on purpose. /boot/config-$(uname -r) and /proc/config.gz
cover a DISTRO kernel. They do not cover a kernel built by a tool like
this one: EL's `make install` does not copy the config to /boot. That
kernel's config is in its build tree, which /lib/modules/<rel>/build still
points at. Falling through to defconfig silently is how you build a kernel
without your root filesystem in it, so that never happens unasked.
skip_own : the tree's .config came from a TRIMMED run and this one is full.
want_full: prefer a full distro config over a running kernel that is trimmed."""
rel = running or os.uname().release
own = os.path.join(tree, ".config")
if mode == "defconfig":
return "defconfig", "", "make defconfig (asked for)"
if mode not in ("auto", "running", "distro"):
path = os.path.expanduser(mode)
if not os.path.isfile(path):
raise ToolError("--config-base %s is not a readable file (expected auto, "
"running, defconfig or a path)" % mode)
return ("gz" if path.endswith(".gz") else "file"), path, path
if mode == "auto" and not skip_own and os.path.isfile(own) and DotConfig(own).valid():
return "file", own, "the .config already in this tree"
chain = ["boot/config-" + rel, "proc/config.gz",
"lib/modules/%s/build/.config" % rel, "lib/modules/%s/config" % rel,
"usr/src/kernels/%s/.config" % rel,
"usr/src/linux-headers-%s/.config" % rel]
others = [c for c in glob.glob(rp(root, "boot/config-*")) if DotConfig(c).valid()]
mine = ""
for c in chain:
path = rp(root, c)
if not (os.path.isfile(path) and os.access(path, os.R_OK)):
continue
if skip_own and os.path.realpath(path) == os.path.realpath(own):
continue
if path.endswith(".gz") or DotConfig(path).valid():
mine = path
break
# A config from the SAME series as the tree lines up symbol for symbol. One
# from a different series loses every option renamed or re-parented between
# them -- carrying 7.2 back to 6.12 drops HYPERV, DMABUF_MOVE_NOTIFY and
# with it GPU peer-to-peer. So a same-series distro config beats the
# running kernel's, unless the user says `--config-base running`.
if mode == "auto" and target_series and (not mine or config_series(mine) != target_series):
close = [c for c in others if config_series(c) == target_series]
if close:
best = richest(close)
return "file", best, (
"%s (a %s config, like this tree%s; --config-base running overrides)"
% (best, target_series,
" -- the running kernel's is %s" % config_series(mine) if mine else ""))
if mode == "auto" and want_full and mine and not mine.endswith(".gz") and others:
best = richest(others)
if DotConfig(mine).count("m") * 2 < DotConfig(best).count("m"):
return "file", best, ("%s (the running kernel's config is a trimmed one; "
"this is the fullest here)" % best)
if mine:
return ("gz" if mine.endswith(".gz") else "file"), mine, \
"%s (the running kernel's)" % mine
if others:
best = richest(others)
return "file", best, \
"%s (NOT the running kernel's -- the fullest distro config found)" % best
raise ToolError("no base config found for %s.\n Looked in /boot, /proc/config.gz, "
"/lib/modules/%s and /usr/src.\n Pass --config-base <file>, or "
"--config-base defconfig if you really mean a bare defconfig."
% (rel, rel))
class Make(object):
"""make, in one tree, with one toolchain and one scrubbed environment.
The toolchain arguments go on EVERY call: Kconfig's cc-option tests run the
compiler, so an olddefconfig without LLVM= resolves the config for gcc,
verify approves it, and the build then quietly re-resolves it for clang."""
def __init__(self, tree, toolchain, jobs):
self.tree, self.tc, self.jobs = tree, toolchain, jobs
self.env = scrubbed_env(toolchain.get("extra_path"))
def argv(self, targets, parallel=False):
cmd = ["make"] + (["-j%d" % self.jobs] if parallel else [])
return cmd + list(self.tc["args"]) + list(targets)
def call(self, targets, extra_env=None, check=True, stdin_data=None):
env = dict(self.env)
env.update(extra_env or {})
return run(self.argv(targets), cwd=self.tree, env=env, check=check,
stdin_data=stdin_data)
def value(self, target):
return (self.call(["-s", target]).stdout or "").strip().splitlines()[-1]
def _find_lld(ci, machine):
cands = [os.path.join(os.path.dirname(ci["path"]), "ld.lld"),
which("ld.lld", path=":".join(SYSTEM_PATH)) or ""]
cands += [os.path.join(os.path.dirname(t["path"]), "ld.lld")
for t in machine.get("toolchains", []) if t["kind"] == "clang"]
return next((c for c in cands if c and os.access(c, os.X_OK)), "")
def resolve_toolchain(opts, machine=None):
"""The user's choices -> one compiler, one set of make arguments, one PATH.
Four independent choices: the COMPILER (--cc, or --toolchain llvm), the
BINUTILS (--binutils auto | gnu | llvm | <dir with ld and as>), the LINKER
(--linker default | bfd | lld) and the PYTHON kbuild's scripts run under.
'auto' binutils means: what was built to go with that compiler -- LLVM's
own for a clang that has the full set beside it, a gcc-toolset's own, and
the system's otherwise."""
machine = machine if isinstance(machine, dict) else {}
if opts["cc"]:
want = os.path.expanduser(opts["cc"])
path = want if os.sep in want else (which(want) or "")
if not path or not os.path.exists(path):
raise ToolError("--cc %s: no such compiler. (Falling back to the system "
"gcc without saying so is how a build ends up made by a "
"compiler nobody chose.)" % opts["cc"])
path = os.path.abspath(path)
elif opts["toolchain"] == "llvm":
prefix = os.path.expanduser(opts["llvm_prefix"])
if prefix and not prefix.endswith("/"):
prefix += "/"
path = (prefix + "clang") if prefix else (which("clang", path=":".join(SYSTEM_PATH)) or "")
else:
path = "/usr/bin/gcc" if os.path.exists("/usr/bin/gcc") else \
(which("gcc", path=":".join(SYSTEM_PATH)) or "")
ci = compiler_info(path) if path else None
if not ci:
raise ToolError("no working compiler at %s" % (path or "/usr/bin/gcc"))
if ci["usable"] == "no":
raise ToolError("%s cannot build a kernel: %s.\n (%s)" % (path, ci["why"], ci["version"]))
if opts["toolchain"] == "llvm" and ci["kind"] != "clang":
raise ToolError("--toolchain llvm, but %s is %s" % (path, ci["version"]))
kind, args, extra = ci["kind"], [], []
binutils = opts.get("binutils", "auto")
llvm = llvm_companions(ci["path"]) if kind == "clang" else None
if binutils == "auto":
binutils = ("llvm" if llvm else "gnu") if kind == "clang" else (own_binutils(path) or "gnu")
if binutils == "llvm":
if kind != "clang":
raise ToolError("--binutils llvm needs clang as the compiler; %s is gcc" % path)
if not llvm:
d = os.path.dirname(ci["path"])
raise ToolError("no complete LLVM tool set beside %s (missing: %s) -- use "
"--binutils gnu" % (path, " ".join(
t for t in LLVM_TOOLS if not os.access(os.path.join(d, t), os.X_OK))))
# HOSTLDFLAGS: tools/lib/bpf is built non-PIC by gcc and linked by an
# ld.lld that defaults to PIE; resolve_btfids dies on R_X86_64_32.
args = [llvm[0], "HOSTLDFLAGS=-no-pie"]
if llvm[1]:
extra.append(llvm[1])
ci["binutils_label"] = "LLVM (%s)" % llvm[0]
else:
if binutils != "gnu":
bdir = os.path.abspath(os.path.expanduser(binutils))
if not all(os.access(os.path.join(bdir, t), os.X_OK) for t in ("ld", "as")):
raise ToolError("--binutils %s: no ld and as in that directory (expected "
"auto, gnu, llvm or a binutils bin directory)" % binutils)
if os.path.realpath(bdir) != os.path.realpath("/usr/bin"):
extra.append(bdir)
sysgcc = ci["real"] == os.path.realpath("/usr/bin/gcc")
if not sysgcc or extra:
# By absolute path. A directory put ahead of /usr/bin for its ld may
# hold a `gcc` of its own, and a bare CC=gcc would silently become it.
args.append("CC=" + ci["path"])
if extra and kind == "gcc":
args.append("HOSTCC=" + ("/usr/bin/gcc" if os.path.exists("/usr/bin/gcc") else ci["path"]))
ci["binutils_label"] = "GNU (%s)" % (extra[0] if extra else "system")
linker = opts.get("linker", "default")
if linker == "lld" and binutils != "llvm":
lld = _find_lld(ci, machine)
if not lld:
raise ToolError("--linker lld: no ld.lld found on this system")
args.append("LD=" + lld)
elif linker == "bfd" and binutils == "llvm":
args.append("LD=ld.bfd")
ci["python"] = ""
if opts.get("python"):
py = os.path.abspath(os.path.expanduser(opts["python"]))
try:
o = (run([py, "-c", "import sys; print(sys.version_info[0])"], check=False,
env=scrubbed_env(), timeout=20).stdout or "").strip()
except ToolError:
o = ""
if o != "3":
raise ToolError("--python %s does not run as a Python 3 in a clean environment"
% opts["python"])
args.append("PYTHON3=" + py)
ci["python"] = py
partner = cxx_partner(ci["path"])
if opts.get("cxx"):
want = os.path.expanduser(opts["cxx"])
cxx = want if os.sep in want else (which(want) or "")
xi = compiler_info(cxx) if cxx and os.path.exists(cxx) else None
if not xi:
raise ToolError("--cxx %s: no such compiler" % opts["cxx"])
if xi["usable"] == "no":
raise ToolError("%s cannot be kbuild's C++ compiler: %s.\n (%s)"
% (cxx, xi["why"], xi["version"]))
cxx = os.path.abspath(cxx)
args.append("HOSTCXX=" + cxx)
if kind == "gcc" and partner and os.path.realpath(cxx) != os.path.realpath(partner):
ci["cxx_note"] = ("GCC plugins are compiled by this against %s's plugin headers; "
"the matching one is %s" % (os.path.basename(ci["path"]), partner))
elif binutils == "llvm":
cxx = partner or "clang++" # kbuild sets HOSTCXX=clang++ itself under LLVM=
elif kind == "gcc" and partner and ci["real"] != os.path.realpath("/usr/bin/gcc"):
# A gcc that is not the system's: ITS g++ builds ITS plugins.
cxx = partner
args.append("HOSTCXX=" + cxx)
else:
cxx = which("g++", path=":".join(extra + SYSTEM_PATH)) or ""
ci["cxx"] = cxx
if opts.get("ccache") == "on":
cc = which("ccache", path=":".join(SYSTEM_PATH))
if not cc:
raise ToolError("--ccache on, but ccache is not installed")
args = [a for a in args if not a.startswith("CC=")] + ["CC=%s %s" % (cc, ci["path"])]
ci["args"], ci["extra_path"] = args, (":".join(extra) or None)
ci["binutils"], ci["linker"], ci["ccache"] = binutils, linker, opts.get("ccache") == "on"
good, why = compiler_works(ci["path"], ci["extra_path"])
if not good:
raise ToolError("%s cannot build an executable in the clean environment the "
"build runs in:\n %s\n If it needs LD_LIBRARY_PATH to find its "
"own libraries, that is the cause." % (ci["path"], why))
return ci
def describe_tools(tc):
"""What the build will REALLY run, asked of the tools themselves in the
environment the build will have. -> [(label, text)]"""
env = scrubbed_env(tc.get("extra_path"))
path = env["PATH"]
def banner(argv):
try:
o = run(argv, check=False, env=env, timeout=20).stdout or ""
except ToolError:
return "NOT FOUND"
return ([l for l in o.splitlines() if l.strip()] or ["?"])[0].strip()[:60]
rows = []
if tc["kind"] == "clang":
rows.append(("assembler", "clang's integrated assembler"))
else:
name = (run([tc["path"], "-print-prog-name=as"], check=False, env=env).stdout or "as").strip()
full = name if os.sep in name else (which(name, path=path) or name)
rows.append(("assembler", "%s (%s)" % (os.path.realpath(full), banner([full, "--version"]))))
ld = next((a[3:] for a in tc["args"] if a.startswith("LD=")), "")
if not ld:
arg = next((a[5:] for a in tc["args"] if a.startswith("LLVM=")), None)
if arg is None:
ld = "ld"
else:
ld = "ld.lld" if arg == "1" else "ld.lld" + arg if arg.startswith("-") else arg + "ld.lld"
full = ld if os.sep in ld else (which(ld, path=path) or ld)
rows.append(("linker", "%s (%s)" % (full, banner([full, "-v"]))))
py = tc.get("python") or which("python3", path=path) or "python3"
rows.append(("python3", "%s (%s)" % (py, banner([py, "--version"]))))
mk = which("make", path=path) or "make"
rows.append(("make", "%s (%s)" % (mk, banner([mk, "--version"]))))
cxx = tc.get("cxx", "")
full = cxx if os.sep in cxx else (which(cxx, path=path) or "") if cxx else ""
rows.append(("c++ (host)", "%s (%s)" % (full, banner([full, "--version"])) if full else
"NONE FOUND -- GCC plugins and xconfig cannot be built"))
if tc.get("cxx_note"):
rows.append(("", tc["cxx_note"]))
if tc.get("ccache"):
rows.append(("ccache", "on"))
tc["_ld"], tc["_python"] = full, py
return rows
REQ_RELEVANT = ("bash", "bc", "bindgen", "binutils", "bison", "Clang/LLVM", "flex", "GNU awk",
"GNU C", "GNU make", "GNU tar", "kmod", "openssl", "pahole", "Python", "Rust")
def parse_requirements(text):
"""The 'Current Minimal Requirements' table in a tree's own
Documentation/process/changes.rst -> [(program, minimum, check command)].
The tree says what IT needs; nothing here hardcodes a version."""
rows, rulers = [], 0
for line in text.splitlines():
if re.match(r"^=+\s+=+\s+=+\s*$", line):
rulers += 1
if rulers == 3:
break
elif rulers == 2:
parts = re.split(r"\s{2,}", line.strip(), maxsplit=2)
if len(parts) == 3:
rows.append(tuple(parts))
return rows
def check_tree_requirements(tree, tc, verbose=False):
"""-> number of blocking problems. Checks the tools that were CHOSEN --
this compiler, this ld, this python -- not whatever answers to `gcc`."""
rows = parse_requirements(read_text(os.path.join(tree, "Documentation/process/changes.rst")))
if not rows:
return 0
if "_ld" not in tc:
describe_tools(tc)
env = scrubbed_env(tc.get("extra_path"))
problems, fine, absent = 0, [], []
for prog, minimum, cmd in rows:
if not prog.startswith(REQ_RELEVANT):
continue # e2fsprogs, GRUB, iptables: runtime, not build
optional = "(optional)" in prog
name = prog.replace("(optional)", "").strip()
argv = shlex.split(cmd.split("||")[0])
if name == "GNU C":
if tc["kind"] != "gcc":
continue
argv = [tc["path"], "--version"]
elif name == "Clang/LLVM":
if tc["kind"] != "clang":
continue
argv, optional = [tc["path"], "--version"], False
elif name == "binutils":
if "lld" in os.path.basename(tc["_ld"]):
continue # LLD answers to the Clang/LLVM row
argv = [tc["_ld"], "-v"]
elif name == "Python":
argv = [tc["_python"], "--version"]
try:
o = run(argv, check=False, env=env, timeout=20)
have = numver(o.stdout or "") if o.returncode == 0 else ()
except ToolError:
have = ()
need = numver(minimum)
if not have:
if optional:
absent.append(name)
else:
problems += 1
bad("%s: not found (this tree needs >= %s; checked with `%s`)"
% (name, minimum, " ".join(argv)))
elif not ver_ge(have, need):
if optional:
absent.append("%s (%s < %s)" % (name, vstr(have), minimum))
else:
problems += 1
bad("%s %s is too old: this tree needs >= %s (`%s`)"
% (name, vstr(have), minimum, " ".join(argv)))
else:
fine.append("%s %s" % (name, vstr(have)))
if verbose:
ok("%-20s %-10s (needs >= %s)" % (name, vstr(have), minimum))
if fine and not verbose:
ok("%d build tools meet this tree's own minimums (changes.rst)" % len(fine))
if absent:
note("optional, not available: %s" % ", ".join(absent))
return problems
COMPRESSORS = (("KERNEL_GZIP", "gzip"), ("KERNEL_BZIP2", "bzip2"), ("KERNEL_LZMA", "lzma"),
("KERNEL_XZ", "xz"), ("KERNEL_LZO", "lzop"), ("KERNEL_LZ4", "lz4"),
("KERNEL_ZSTD", "zstd"), ("MODULE_COMPRESS_GZIP", "gzip"),
("MODULE_COMPRESS_XZ", "xz"), ("MODULE_COMPRESS_ZSTD", "zstd"))
def check_compressors(cfg, path):
"""The config picks how the image and the modules are compressed; the binary
has to exist. Found here, this costs a second. Found by the build, it is
the very last step, after every object has been compiled."""
missing = [(sym, tool) for sym, tool in COMPRESSORS
if cfg.get(sym) == "y" and not which(tool, path=path)]
for sym, tool in missing:
bad("CONFIG_%s=y, but there is no `%s` on this machine" % (sym, tool))
return len(missing)
HOLD = ".config.elkt-hold"
HOLD_NONE = ".config.elkt-hold.none"
STAGING = ".config.elkt-staging"
def lsmod_file(path, root="/"):
with open(path, "w") as f:
f.write("Module Size Used by\n")
for line in read_text(rp(root, "proc/modules")).splitlines():
p = line.split()
if len(p) >= 3:
f.write("%-23s %s %s\n" % (p[0], p[1], p[2]))
def recover_held_config(tree):
"""Undo whatever a run killed inside trim_to_loaded() left behind. Called
before ANYTHING reads, backs up or cleans the tree's .config."""
real = os.path.join(tree, ".config")
hold, none = os.path.join(tree, HOLD), os.path.join(tree, HOLD_NONE)
if os.path.exists(hold):
os.replace(hold, real)
note("restored the .config an interrupted run had set aside")
elif os.path.exists(none) and os.path.exists(real):
# That run had NO .config of its own: what sits there is its unverified
# staging copy, and `auto` would adopt it as the user's.
os.unlink(real)
note("removed an unverified .config left by an interrupted run")
if os.path.exists(none):
os.unlink(none)
def trim_to_loaded(mk, staging):
"""`make localmodconfig` against the staged config. That target hardcodes
'.config' (streamline_config.pl ignores KCONFIG_CONFIG), so the tree's real
.config is held aside for the duration and put back in a finally -- and a
marker says so when there was none, for the kill the finally never sees."""
tree = mk.tree
real, hold = os.path.join(tree, ".config"), os.path.join(tree, HOLD)
none = os.path.join(tree, HOLD_NONE)
lsmod = os.path.join(tree, ".elkt-lsmod")
had = os.path.exists(real)
if had:
os.replace(real, hold)
else:
open(none, "w").close()
try:
shutil.copyfile(staging, real)
lsmod_file(lsmod)
# oldconfig underneath asks about every new symbol; feed it Enters.
mk.call(["localmodconfig"], extra_env={"LSMOD": lsmod}, stdin_data="\n" * 20000)
os.replace(real, staging)
finally:
if os.path.exists(lsmod):
os.unlink(lsmod)
if had:
os.replace(hold, real)
else:
if os.path.exists(real):
os.unlink(real)
os.unlink(none)
def configure(tree, opts, machine, mk, vendors):
"""Everything happens in a STAGING file via KCONFIG_CONFIG. The tree's
.config is replaced only by a config that passed verification, so a failed
run leaves nothing behind for the next run to adopt as 'the user's'."""
recover_held_config(tree)
staging = os.path.join(tree, STAGING)
kenv = {"KCONFIG_CONFIG": STAGING}
kind, path, label = find_base_config(tree, opts["config_base"],
target_series=series_of(mk.value("kernelversion")),
skip_own=opts.get("_skip_own", False),
want_full=opts["modules"] == "full")
info("base : %s" % label)
if kind == "defconfig":
if os.path.exists(staging):
os.unlink(staging)
mk.call(["defconfig"], extra_env=kenv)
elif kind == "gz":
with gzip.open(path, "rb") as src, open(staging, "wb") as dst:
shutil.copyfileobj(src, dst)
else:
shutil.copyfile(path, staging)
if not DotConfig(staging).valid():
raise ToolError("the base config %s has no CONFIG_ lines in it" % label)
mk.call(["olddefconfig"], extra_env=kenv)
before = DotConfig(staging)
if opts["modules"] == "trimmed":
trim_to_loaded(mk, staging)
after = DotConfig(staging)
info("trimmed : %d -> %d modules (to what is loaded now)"
% (before.count("m"), after.count("m")))
symbols = kconfig_symbols(tree, machine["arch"])
reqs = build_requests(opts, vendors, machine)
results = []
for attempt in (1, 2):
cfg = DotConfig(staging)
for r in reqs:
if r.sym in symbols:
cfg.set(r.sym, r.rendered(cfg.get(r.sym), attempt))
cfg.write()
mk.call(["olddefconfig"], extra_env=kenv)
results = verify_requests(DotConfig(staging), reqs, symbols)
# A second pass catches a symbol whose prompt only became visible once
# the first pass had switched its dependency on.
if all(s == "ok" or s == "absent" for _, _, s in results):
break
return staging, reqs, results, symbols
def report_verify(results):
fails = 0
for r, got, status in results:
shown = "CONFIG_%-28s" % r.sym
gots = "<absent>" if got is None else got
if status == "ok":
if r.want != "n" and r.level != "optional":
ok("%s = %s" % (shown, gots if r.want == "m|y" else r.shown() or '""'))
elif status == "absent":
if r.want != "n":
note("%s not in this kernel version%s"
% (shown, " (%s)" % r.why if r.why else ""))
elif status == "fail":
fails += 1
bad("%s want %s, got %s%s" % (shown, r.shown() or '""', gots,
" -- " + r.why if r.why else ""))
elif status == "warn":
warn("CONFIG_%s: wanted %s, got %s%s"
% (r.sym, r.shown() or '""', gots, " (%s)" % r.why if r.why else ""))
elif r.want != "n":
note("%s wanted %s, got %s" % (shown, r.shown(), gots))
return fails
def drop_staging_litter(staging):
for extra in (staging + ".old", staging + ".tmp"):
if os.path.exists(extra):
os.unlink(extra)
def commit_config(tree, staging):
real = os.path.join(tree, ".config")
if os.path.exists(real):
shutil.copy2(real, os.path.join(tree, ".config.elkt-prev"))
os.replace(staging, real)
drop_staging_litter(staging)
# ------------------------------------------------------------------------- state
def state_dir(opts):
d = os.path.abspath(os.path.expanduser(opts.get("state_dir") or os.path.join(
os.environ.get("XDG_STATE_HOME") or "~/.local/state", TOOL)))
os.makedirs(d, exist_ok=True)
return d
class State(object):
"""One INI section per SOURCE TREE, flushed whole and atomically on every
change, so the record survives the kill -9 it exists to detect."""
def __init__(self, path):
self.path = path
self.cp = configparser.RawConfigParser()
self.cp.optionxform = str
if os.path.isfile(path):
try:
self.cp.read(path)
except configparser.Error:
warn("%s is unreadable -- starting a fresh state file" % path)
self.cp = configparser.RawConfigParser()
def get(self, tree):
return dict(self.cp.items(tree)) if self.cp.has_section(tree) else {}
def trees(self):
return self.cp.sections()
def update(self, tree, **kv):
# Lock, RE-READ, apply, write. Flushing the snapshot taken at start-up
# would let a second instance, building another tree, write 'building'
# back over this tree's 'built' -- and the next run would mrproper it.
with open(self.path + ".lock", "w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
fresh = configparser.RawConfigParser()
fresh.optionxform = str
try:
fresh.read(self.path)
except configparser.Error:
fresh = configparser.RawConfigParser()
fresh.optionxform = str
if not fresh.has_section(tree):
fresh.add_section(tree)
for k, v in kv.items():
fresh.set(tree, k, str(v))
tmp = "%s.tmp.%d" % (self.path, os.getpid())
with open(tmp, "w") as f:
f.write("# %s -- build state, one [section] per source tree. "
"Safe to delete.\n" % TOOL)
fresh.write(f)
os.replace(tmp, self.path)
self.cp = fresh
def now():
return time.strftime("%Y-%m-%dT%H:%M:%S%z")
def in_service(tree, root="/"):
"""Releases whose /lib/modules/<rel>/build is THIS tree. DKMS compiles
against that link; mrproper here breaks every out-of-tree build for an
installed -- possibly the running -- kernel until a rebuild succeeds."""
rels = []
for link in glob.glob(rp(root, "lib/modules/*/build")):
if os.path.realpath(link) == os.path.realpath(tree):
rels.append(os.path.basename(os.path.dirname(link)))
return sorted(rels)
def decide_clean(mode, has_objects, status, stage, serving):
"""-> (clean?, why). Raises when asked to clean a tree that is in service.
auto cleans in exactly one case: a run that recorded 'building' and never
recorded an end -- kill -9, OOM, power loss -- where make had no chance to
delete a half-written object. A failure we SAW (compile error, ^C) left
the tree consistent; make removes its partial target and kbuild's .cmd
files decide the rest, so the retry is incremental. And a failure AFTER
the build -- a sudo prompt that timed out overnight -- says nothing about
the tree at all; the old script answered it with mrproper."""
if mode not in ("auto", "always", "never", "force"):
raise ToolError("--clean must be auto, always, never or force")
if mode == "never":
return False, "--clean never"
if not has_objects and mode == "auto":
return False, "no object files present -- nothing to clean"
want, why = False, ""
if mode in ("always", "force"):
want, why = True, "--clean " + mode
elif status == "building":
want, why = True, ("the last run recorded 'building' and never recorded an "
"end (kill -9, OOM or power loss)")
elif status == "configuring":
return False, "the last run died while configuring; no object was touched"
elif status == "failed":
return False, ("the last run failed at stage '%s' where make could tidy up; "
"retrying incrementally" % (stage or "?"))
elif status in ("built", "installing", "installed", "configured"):
return False, "the last run finished cleanly (%s); incremental is correct" % status
else:
return False, "objects present, no record of them -- building incrementally"
if want and serving and mode != "force":
msg = ("this tree is the DKMS build directory of installed kernel(s) %s; "
"cleaning it would break out-of-tree module builds for them"
% ", ".join(serving))
if mode == "always":
raise ToolError(msg + ".\n Use --clean force if that is really intended.")
return False, msg + " -- NOT cleaning"
return want, why
def clean_tree(mk, tree, restore=True):
"""mrproper, without losing the configuration to it."""
real = os.path.join(tree, ".config")
keepas = None
if os.path.isfile(real):
# The FIRST backup keeps the plain name: by the second clean .config
# may no longer be the user's original.
keepas = os.path.join(tree, ".config.before-clean")
if os.path.exists(keepas):
keepas += "." + time.strftime("%Y%m%d-%H%M%S")
shutil.copy2(real, keepas)
info("saved : .config -> %s" % os.path.basename(keepas))
try:
mk.call(["mrproper"])
finally:
# Cleaning is about build PRODUCTS; mrproper deleting .config is
# incidental. Restore the copy made a moment ago -- by ITS name, not the
# fixed one, which on a second clean is an OLDER config -- and in a
# finally, so an interrupted mrproper does not lose it either.
if keepas and restore and not os.path.exists(real):
shutil.copy2(keepas, real)
# ------------------------------------------------------------------------- build
STEP = re.compile(r"^\s{2}([A-Z][A-Z0-9_.]*(?: \[M\])?)\s+(\S+)")
# 'error:' with the colon -- the bare word is also a file name (linux/can/error.h
# goes past twice in every build).
ERRORISH = re.compile(r"(\berror:|\*\*\* |undefined reference|No rule to make target|"
r"No space left|Permission denied|cannot find -l|Killed\b)")
def error_summary(log_path, limit=12):
"""The FIRST real errors. The tail of a -j24 build is twenty-three other
jobs finishing; the line that matters is hundreds above it."""
hits = []
try:
with open(log_path, "r", errors="replace") as f:
for n, line in enumerate(f, 1):
if ERRORISH.search(line) and "warning:" not in line:
hits.append("%7d: %s" % (n, line.rstrip()))
if len(hits) >= limit:
break
except OSError:
pass
return hits
def fit_line(text, cols=None):
"""A line redrawn with \\r has to fit on ONE terminal row. \\r returns to the
start of the LAST wrapped row only, so a longer line leaves a copy of itself
behind on every redraw -- a hundred-character status in an 80-column
terminal scrolled the whole build past as thousands of fragments."""
cols = cols or shutil.get_terminal_size((80, 24)).columns
return text[:max(cols - 1, 20)]
def run_build(mk, log_path, verbose=False, targets=(), extra_env=None):
if os.path.exists(log_path):
os.replace(log_path, log_path + ".prev")
started = time.time()
steps = warnings = shown_warn = 0
last, beat, drawn = "", started, 0.0
env = dict(mk.env)
env.update(extra_env or {})
proc = subprocess.Popen(mk.argv(list(targets), parallel=True), cwd=mk.tree, env=env,
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, universal_newlines=True,
errors="replace", bufsize=1)
def status_line():
el = int(time.time() - started)
return fit_line(" [%02d:%02d:%02d] %6d steps %s" % (el // 3600, el // 60 % 60,
el % 60, steps, last))
try:
with open(log_path, "w") as log:
for line in proc.stdout:
log.write(line)
text = line.rstrip("\n")
m = STEP.match(text)
if m:
steps += 1
last = "%s %s" % (m.group(1), m.group(2))
if verbose:
out(text)
continue
important = False
if "warning:" in text:
warnings += 1
shown_warn += 1
important = shown_warn <= 15
if shown_warn == 16:
text, important = " (further warnings go to the log only)", True
elif ERRORISH.search(text) or text.startswith("Kernel:"):
important = True
if important:
if UI.tty:
sys.stdout.write("\r\033[K")
out(text)
if UI.tty:
if important or time.time() - drawn >= 0.1: # ten redraws a second is plenty
drawn = time.time()
sys.stdout.write("\r\033[K" + status_line())
sys.stdout.flush()
elif time.time() - beat >= 120:
beat = time.time()
out(status_line())
rc = proc.wait()
except BaseException:
# ^C reached make too (same process group); SIGTERM, SIGHUP and our own
# errors (disk full on the log) did not. Ask it to stop -- make removes
# its partial targets on SIGTERM -- and KEEP DRAINING the pipe, or its
# parting output blocks it until the timeout.
try:
proc.terminate()
except OSError:
pass
try:
proc.communicate(timeout=60)
except (subprocess.TimeoutExpired, ValueError):
proc.kill()
proc.wait()
raise
finally:
if UI.tty:
sys.stdout.write("\r\033[K")
return rc, {"steps": steps, "warnings": warnings, "seconds": int(time.time() - started)}
def module_names(path):
names = set()
for line in read_text(path).splitlines():
base = os.path.basename(line.strip())
if base:
names.add(re.sub(r"\.(ko|o)(\.\w+)?$", "", base).replace("-", "_"))
return names
def module_coverage(tree, root="/"):
"""Every module loaded on this machine now, checked against what the build
produced (as .ko or built-in). The general form of 'will this boot HERE':
it would have caught a defconfig kernel with no XFS before any reboot did."""
built = module_names(os.path.join(tree, "modules.order")) | \
module_names(os.path.join(tree, "modules.builtin"))
missing, oot = [], []
for line in read_text(rp(root, "proc/modules")).splitlines():
name = line.split()[0] if line.split() else ""
if not name or name in built:
continue
taint = read_text(rp(root, "sys/module/%s/taint" % name)).strip()
(oot if "O" in taint or "E" in taint else missing).append(name)
loaded = len(read_text(rp(root, "proc/modules")).splitlines())
return {"loaded": loaded, "missing": sorted(missing), "out_of_tree": sorted(oot)}
OOT_C = """#include <linux/module.h>
#include <linux/init.h>
static int __init elkt_probe_init(void) { return 0; }
static void __exit elkt_probe_exit(void) { }
module_init(elkt_probe_init);
module_exit(elkt_probe_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("enterprise_linux_kernel_tool out-of-tree build probe");
"""
def oot_probe(tree, env, make_args, jobs=4):
"""Compile a ten-line module against the tree, the way DKMS will: through
`make -C <tree> M=<dir>`. -> (ok, vermagic or the failing output)."""
# kbuild rejects M= paths with spaces, colons or %; $TMPDIR may have them,
# and a refusal here would read as "DKMS will not compile".
base = None if not re.search(r"[\s:%]", tempfile.gettempdir()) else "/tmp"
d = tempfile.mkdtemp(prefix="elkt-oot-", dir=base)
try:
with open(os.path.join(d, "elkt_probe.c"), "w") as f:
f.write(OOT_C)
with open(os.path.join(d, "Makefile"), "w") as f:
f.write("obj-m := elkt_probe.o\n")
p = run(["make", "-j%d" % jobs, "-C", tree, "M=" + d] + list(make_args) + ["modules"],
env=env, check=False, timeout=900)
ko = os.path.join(d, "elkt_probe.ko")
if p.returncode != 0 or not os.path.isfile(ko):
lines = [l for l in (p.stdout or "").splitlines() if l.strip()]
return False, "\n".join(lines[-12:])
vm = ""
modinfo = which("modinfo", path=":".join(SYSTEM_PATH))
if modinfo:
vm = (run([modinfo, "-F", "vermagic", ko], check=False).stdout or "").strip()
return True, vm
finally:
shutil.rmtree(d, ignore_errors=True)
def dkms_make_args(tree, dkms_script=None):
"""What THIS machine's dkms adds to its make command for this tree: None
without dkms, [] for an older one. dkms 3.x reads the kernel's config and,
for a clang-built kernel, appends LLVM=1 -- so it builds with the clang on
ROOT's PATH, which need not be the clang that built the kernel."""
script = dkms_script or which("dkms", path=":".join(SYSTEM_PATH))
if not script:
return None
if "CONFIG_CC_IS_CLANG" in read_text(script) and \
DotConfig(os.path.join(tree, ".config")).get("CC_IS_CLANG") == "y":
return ["LLVM=1"]
return []
def dkms_check(tree, toolchain, machine, release=""):
"""Two questions, kept apart. Will DKMS compile against this tree -- run
the way THIS dkms runs it? And will a PLAIN `make -C <tree> M=...`, which
is how Intel's sep5, VirtualBox and vendor installers build? -> False only
when something on this machine actually NEEDS the one that fails: a DKMS
module that will be built for this release, or an out-of-tree module loaded
right now. A deliberate choice of compiler is not an error by itself."""
say("Out-of-tree (DKMS) build check")
if not os.path.isfile(os.path.join(tree, "Module.symvers")):
note("Module.symvers is missing -- the tree is not fully built; modpost "
"will complain, the compile itself is still a fair test")
if read_text(os.path.join(tree, PIN_FILE)).startswith(PIN_MARK):
note("this tree pins its toolchain (%s): whatever a caller passes, builds "
"through it use %s" % (PIN_FILE, toolchain["path"]))
env = dkms_env(machine["pkg_manager"])
extra = dkms_make_args(tree)
plain_ok, plain_out = oot_probe(tree, env, [])
dkms_ok, dkms_out = oot_probe(tree, env, extra) if extra else (plain_ok, plain_out)
will_build = [m for m in machine.get("dkms", [])
if release and dkms_will_build(m["limits"], release)[0]]
loaded_oot = module_coverage(tree)["out_of_tree"]
needed = bool(will_build or loaded_oot)
how = ("dkms here adds %s for a clang-built kernel: ROOT's clang, not necessarily yours"
% " ".join(extra)) if extra else "root's PATH, system compiler"
def show(text):
for l in text.splitlines():
if "error" in l.lower() or "***" in l or "You are using" in l:
info(" | " + l.strip()[:150])
if extra is None:
note("no dkms on this machine")
elif dkms_ok:
ok("a module compiles the way dkms will run it (%s)" % how)
if release and dkms_out and not dkms_out.startswith(release + " "):
warn("vermagic '%s' does not start with %s" % (dkms_out, release))
else:
(bad if needed else warn)("a module does NOT compile the way dkms will run it (%s):" % how)
show(dkms_out)
if plain_ok:
if extra is None or extra:
ok("a plain `make -C <tree> M=...` compiles too")
else:
warn("a PLAIN out-of-tree build fails -- `make -C /lib/modules/%s/build M=...`, the way "
"Intel's sep5, VirtualBox and vendor installers build. It runs gcc against a "
"kernel built by %s:" % (release or "<release>", toolchain["kind"]))
show(plain_out)
if not (dkms_ok and plain_ok) and toolchain["args"]:
good2, _ = oot_probe(tree, scrubbed_env(toolchain.get("extra_path")), toolchain["args"])
if good2:
envargs = [shlex.quote(a) for a in toolchain["args"]
if a.startswith(("CC=", "LLVM=", "LD="))]
if toolchain.get("extra_path"):
envargs.insert(0, shlex.quote("PATH=%s:/usr/sbin:/usr/bin" % toolchain["extra_path"]))
info("The kernel is fine: the probe compiles with its own toolchain. Put it in the")
info("environment of whatever builds the module -- kbuild reads it from there:")
info(" sudo env %s <the build command, e.g. dkms autoinstall -k %s>"
% (" ".join(envargs), release or "<release>"))
if not needed and not (dkms_ok and plain_ok):
note("nothing here needs it today -- no DKMS module will be built for %s and no "
"out-of-tree module is loaded -- so this does not block an install"
% (release or "this kernel"))
elif needed and not dkms_ok:
bad("needed by: %s" % ", ".join(["%s/%s" % (m["name"], m["version"]) for m in will_build]
+ loaded_oot))
for m in machine.get("dkms", []):
if not release:
continue
will, why = dkms_will_build(m["limits"], release)
label = "%s/%s" % (m["name"], m["version"])
if not will:
note("%s: DKMS will skip %s cleanly (%s)" % (label, release, why))
if m["name"] == "amdgpu":
info(" the in-tree amdgpu built here serves the GPU instead")
elif m["name"] == "amdgpu":
warn("%s has no kernel cap: DKMS will try to build it for %s. Vendor "
"amdgpu-dkms targets older enterprise kernels; against a newer one "
"it usually fails, noisily but harmlessly -- the in-tree driver is "
"the newer of the two. To make DKMS skip it:\n"
" echo 'BUILD_EXCLUSIVE_KERNEL_MAX=\"%s.999\"' | sudo tee -a "
"/usr/src/%s-%s/dkms.conf\n"
" (DKMS skips kernels that sort ABOVE the cap, and 6.12.5 sorts above "
"'6.12' -- hence .999, to keep every release of the series it works on.)"
% (label, release,
series_of(max([k for k, _ in m["kernels"]] or [machine["running"]],
key=version_key)), m["name"], m["version"]))
else:
note("%s: DKMS will rebuild it for %s at install time" % (label, release))
return dkms_ok or not needed
def post_build_checks(tree, toolchain, machine, vendors, release):
say("Post-build checks")
problems = 0
ok("kernel release: %s" % release)
cov = module_coverage(tree)
if cov["missing"]:
share = 100.0 * len(cov["missing"]) / max(cov["loaded"], 1)
warn("%d of the %d modules loaded right now have no counterpart in this build:\n"
" %s%s" % (len(cov["missing"]), cov["loaded"],
" ".join(cov["missing"][:24]),
" ..." if len(cov["missing"]) > 24 else ""))
if share > 25:
bad("that is %.0f%% of them -- this config was not made for this machine" % share)
problems += 1
else:
ok("all %d modules loaded right now exist in this build" % cov["loaded"])
if cov["out_of_tree"]:
note("out-of-tree modules loaded now (DKMS must rebuild them): %s"
% " ".join(cov["out_of_tree"]))
tcfg = DotConfig(os.path.join(tree, ".config"))
if tcfg.get("DEBUG_INFO_BTF") != "y" and tcfg.get("BPF_LSM") == "y":
note("BTF is off but BPF_LSM is on: systemd will log 'bpf-restrict-fs: Failed "
"to load BPF object' at boot, and bpftrace / CO-RE tools will not run. "
"--debug-info btf fixes that, for a slower build.")
built = module_names(os.path.join(tree, "modules.order"))
for vendor, mod in (("amd", "amdgpu"), ("nvidia", "nouveau"), ("intel", "i915")):
if vendor in vendors:
if mod in built:
ok("%s built as a module" % mod)
else:
bad("%s was requested as a module and was not built" % mod)
problems += 1
if not dkms_check(tree, toolchain, machine, release):
problems += 1
return problems
# ------------------------------------------------------------ the toolchain pin
PIN_FILE = "GNUmakefile"
PIN_MARK = "# elkt-toolchain-pin"
PINNED_VARS = ("LLVM", "CC", "LD", "AR", "NM", "OBJCOPY", "OBJDUMP", "READELF", "STRIP",
"HOSTCC", "HOSTCXX", "HOSTAR", "HOSTLD")
def pin_text(tc, sign=False):
"""A GNUmakefile that makes THIS TREE build everything that comes through it
with the toolchain the kernel was built with -- '' when that is simply the
system's.
GNU make reads GNUmakefile before Makefile. So `make -C
/lib/modules/<rel>/build M=...` -- DKMS, Intel's sep5, VirtualBox, a vendor
installer -- lands here first, whatever it passes or forgets to pass. That
is the whole problem with a kernel built by a private clang: root's builds
fall back to gcc, or (dkms 3.x, helpfully adding LLVM=1) to root's OLDER
clang, and both choke on flags the kernel's config recorded.
Two things make it stick. `override` beats a command-line LLVM=1 in this
make. But kbuild immediately re-invokes itself with `-f Makefile`, which
does NOT read this file -- so the caller's definitions are taken out of
MAKEOVERRIDES and the pinned ones put in, which is how make hands
command-line variables down. ELKT_NO_PIN=1 switches it all off, and it
switches itself off where the compiler does not exist (another machine)."""
pins = []
for a in tc["args"]:
k, _, v = a.partition("=")
if k in PINNED_VARS:
pins.append((k, tc["path"] if k == "CC" and tc.get("ccache") else v))
extra = tc.get("extra_path") or ""
if not pins and not extra and not sign:
return ""
esc = lambda v: v.replace("$", "$$")
lines = [PIN_MARK + " -- written by %s; safe to delete." % TOOL,
"# This tree was built with: %s" % tc["version"]]
if pins or extra:
lines += ["# Every build that comes through this directory uses the same toolchain.",
"# Switch off with ELKT_NO_PIN=1 in the environment.",
"ifeq ($(ELKT_NO_PIN),)",
"ifneq ($(wildcard %s),)" % esc(tc["path"]),
"MAKEOVERRIDES := $(filter-out $(addsuffix =%%,%s),$(MAKEOVERRIDES))"
% " ".join(PINNED_VARS)]
for k, v in pins:
lines += ["override %s := %s" % (k, esc(v)), "MAKEOVERRIDES += %s=%s" % (k, esc(v))]
# The ones NOT pinned must not arrive from the caller's ENVIRONMENT either: a
# shell that exports LLVM= (found on the machine this was written on) would
# turn a gcc-built tree's module builds into clang ones. unexport keeps them
# out of the sub-make, where the work happens; kbuild's own defaults apply.
if pins or extra:
loose = [v for v in PINNED_VARS if v not in [k for k, _ in pins]]
lines.append("unexport %s" % " ".join(loose))
if extra:
lines.append("export PATH := %s:$(PATH)" % esc(extra))
lines += ["endif", "endif"]
lines.append("include Makefile")
if sign:
# WHY HERE. `make binrpm-pkg` / `bindeb-pkg` run a plain, goal-less `make`
# in this directory as their build step -- with a NEW build number
# (scripts/build-version increments on every call), so vmlinux is relinked
# and bzImage rewritten: a signature put there beforehand is gone, and the
# package ships an UNSIGNED kernel. Measured, not assumed. So the
# goal-less make itself ends by signing. Builds with a goal, and
# out-of-tree builds (M=), are left exactly as they were.
tool = esc(os.path.abspath(__file__))
lines += ["# Secure Boot: a goal-less `make` here ends by signing the image again.",
"ifeq ($(KBUILD_EXTMOD)$(M)$(MAKECMDGOALS)$(ELKT_NO_SIGN),)",
"ifneq ($(wildcard %s),)" % tool,
".DEFAULT_GOAL := elkt-all",
".PHONY: elkt-all",
"elkt-all: __all",
"\t@%s -I %s sign --source $(CURDIR)" % (PLATFORM_PYTHON, tool),
"endif", "endif"]
lines.append("")
return "\n".join(lines)
def pin_toolchain(tree, tc, sign=False):
"""Write, refresh or REMOVE the pin -- on every build, because a stale pin
would quietly overrule the compiler chosen this time. -> True when pinned."""
path = os.path.join(tree, PIN_FILE)
ours = os.path.isfile(path) and read_text(path).startswith(PIN_MARK)
if os.path.exists(path) and not ours:
warn("%s exists and is not ours -- leaving it alone; the toolchain is NOT pinned" % path)
return False
text = pin_text(tc, sign)
if not text:
if ours:
os.unlink(path)
return False
tmp = "%s.tmp.%d" % (path, os.getpid())
with open(tmp, "w") as f:
f.write(text)
os.replace(tmp, path)
return True
# ------------------------------------------------------------------- secure boot
def keys_dir():
return os.path.join(os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config"),
TOOL, "keys")
def signing_key(create=True):
"""YOUR Secure Boot key: made once, kept, used for every image. Persistent on
purpose -- a machine enrols a CERTIFICATE in MOK once, and everything signed
by that key boots on it afterwards. That is also what you hand out with a
kernel other people download: the .der, never the .key.
RSA 2048 / SHA-256, as the distro and Microsoft Secure Boot certificates are:
shim takes larger keys, some firmware does not."""
base = os.path.join(keys_dir(), "secureboot")
k = {"key": base + ".key", "crt": base + ".crt", "der": base + ".der"}
if all(os.path.isfile(v) for v in k.values()):
return k
if not create:
return None
os.makedirs(keys_dir(), exist_ok=True)
os.chmod(keys_dir(), 0o700)
who = re.sub(r"[^\w@.-]", "_", "%s@%s" % (os.environ.get("USER", "builder"), os.uname().nodename))
old = os.umask(0o077)
try:
run(["openssl", "req", "-new", "-x509", "-newkey", "rsa:2048", "-sha256", "-nodes",
"-days", "3650", "-subj", "/CN=%s kernel signing key/" % who,
"-addext", "basicConstraints=critical,CA:FALSE",
"-addext", "keyUsage=critical,digitalSignature",
"-addext", "extendedKeyUsage=codeSigning",
"-keyout", k["key"], "-out", k["crt"]], env=scrubbed_env())
run(["openssl", "x509", "-in", k["crt"], "-outform", "DER", "-out", k["der"]],
env=scrubbed_env())
finally:
os.umask(old)
os.chmod(k["der"], 0o644)
os.chmod(k["crt"], 0o644)
info("made a Secure Boot signing key: %s (keep the .key private; back it up)" % k["key"])
return k
def key_cn(k):
o = run(["openssl", "x509", "-in", k["crt"], "-noout", "-subject", "-nameopt", "multiline"],
env=scrubbed_env(), check=False).stdout or ""
m = re.search(r"commonName\s*=\s*(.+)", o)
return m.group(1).strip() if m else ""
def image_path(tree, arch=None):
arch = arch or os.uname().machine
rel = {"x86_64": "arch/x86/boot/bzImage"}.get(arch)
return os.path.join(tree, rel) if rel else ""
def signing_tools():
path = ":".join(SYSTEM_PATH)
if which("sbsign", path=path):
return "sbsign"
if all(which(t, path=path) for t in ("pesign", "certutil", "pk12util", "openssl")):
return "pesign"
return ""
def image_signers(image):
path = ":".join(SYSTEM_PATH)
if which("pesign", path=path):
return run(["pesign", "-S", "-i", image], check=False, env=scrubbed_env()).stdout or ""
if which("sbverify", path=path):
return run(["sbverify", "--list", image], check=False, env=scrubbed_env()).stdout or ""
return ""
def sign_image(tree, k):
"""-> 'signed' | 'already'. The signed image REPLACES bzImage (the unsigned
one is kept beside it), because `make install` installs that file."""
img = image_path(tree)
if not img:
raise ToolError("image signing is implemented for x86_64 only, so far")
if not os.path.isfile(img):
raise ToolError("%s does not exist -- build first" % img)
cn = key_cn(k)
if cn and cn in image_signers(img):
return "already"
tool = signing_tools()
if not tool:
raise ToolError("no signing tool: install sbsigntools, or pesign + nss-tools")
signed, env = img + ".signed", scrubbed_env()
if os.path.exists(signed):
os.unlink(signed)
if tool == "sbsign":
run(["sbsign", "--key", k["key"], "--cert", k["crt"], "--output", signed, img], env=env)
else:
# pesign signs from an NSS database only: a throw-away one, made for this.
db = tempfile.mkdtemp(prefix="elkt-nss-")
try:
p12 = os.path.join(db, "k.p12")
run(["certutil", "-N", "-d", "sql:" + db, "--empty-password"], env=env)
run(["openssl", "pkcs12", "-export", "-inkey", k["key"], "-in", k["crt"],
"-name", "elkt", "-out", p12, "-passout", "pass:"], env=env)
run(["pk12util", "-d", "sql:" + db, "-i", p12, "-W", ""], env=env)
run(["pesign", "-n", "sql:" + db, "-c", "elkt", "-s", "-i", img, "-o", signed], env=env)
finally:
shutil.rmtree(db, ignore_errors=True)
if not cn or cn not in image_signers(signed):
raise ToolError("signed %s but cannot find the signature in it -- NOT using it" % img)
shutil.copy2(img, img + ".unsigned")
os.replace(signed, img)
return "signed"
def mok_enrolled(k):
"""True / False / None (cannot tell: no mokutil, or not an EFI boot)."""
mok = which("mokutil", path=":".join(SYSTEM_PATH))
if not mok or not os.path.isdir("/sys/firmware/efi"):
return None
o = run([mok, "--test-key", k["der"]], check=False, env=scrubbed_env()).stdout or ""
return True if "already enrolled" in o else False if "not enrolled" in o else None
def secure_boot_sign(tree, machine):
say("Secure Boot signing")
k = signing_key()
how = sign_image(tree, k)
ok("%s %s with '%s'" % (os.path.relpath(image_path(tree), tree),
"is signed" if how == "signed" else "was already signed", key_cn(k)))
enrolled = mok_enrolled(k)
if enrolled:
ok("that certificate is enrolled in this machine's MOK list")
else:
note("to boot it with Secure Boot ON, a machine must trust the certificate -- once:")
info(" sudo mokutil --import %s (choose a one-time password)" % k["der"])
info(" reboot; in the blue MOK Manager screen: Enroll MOK, that password")
info(" Anyone you give this kernel to needs that .der file too -- never the .key.")
return k
# ----------------------------------------------------------------------- install
class SudoKeepalive(object):
"""Ask for the password ONCE, up front, and keep the ticket warm. A prompt
that appears at the end of an unattended build times out after five
minutes, and the install 'fails' at 3 a.m. for no reason."""
def __init__(self):
self._stop = threading.Event()
def start(self):
if run(["sudo", "-v"], interactive=True, check=False).returncode != 0:
raise ToolError("sudo authentication failed")
t = threading.Thread(target=self._loop)
t.daemon = True
t.start()
def _loop(self):
while not self._stop.wait(50):
subprocess.call(["sudo", "-n", "-v"], stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def stop(self):
self._stop.set()
def sudo(cmd, check=True, capture=False, dry=False):
if dry:
info("would run: sudo " + " ".join(shlex.quote(c) for c in cmd))
return None
return run(["sudo"] + cmd, check=check, capture=capture, interactive=not capture)
def install_preflight(machine, opts, release="", tree=""):
"""Everything that can refuse an install, checked BEFORE the long part when
`build --install` is used, and again with the real release afterwards."""
if release and release == machine["running"] and not opts["yes"]:
raise ToolError("the release being installed (%s) is the kernel running now.\n"
" Installing over it replaces the modules of a live kernel. "
"Change --localversion, or pass --yes to do it anyway." % release)
if machine["secure_boot"] == "on":
k = signing_key(create=False)
img = image_path(tree) if tree else ""
signed = bool(k and img and os.path.isfile(img) and key_cn(k) in image_signers(img))
if tree and not signed and not opts["allow_unsigned"]:
raise ToolError("Secure Boot is ON here and this image is not signed: shim will "
"refuse it.\n Run `%s.py sign --source %s`, or pass --allow-unsigned."
% (TOOL, tree))
if k and mok_enrolled(k) is False and not opts["yes"]:
raise ToolError("Secure Boot is ON here and your signing certificate is not enrolled:"
"\n sudo mokutil --import %s\n then reboot and enrol it in the "
"MOK Manager. --yes installs anyway (it will not boot until then)."
% k["der"])
ok("Secure Boot here: %s" % machine["secure_boot"])
if not machine["boot"]["initramfs"] and not opts["yes"]:
raise ToolError("no initramfs generator here (dracut, update-initramfs, mkinitcpio).\n"
" `make install` would install a kernel with no initramfs, and a root "
"on %s%s needs modules from one. Install one, or --yes to go ahead."
% (machine["storage"]["rootfs"] or "this filesystem",
" over " + ",".join(machine["storage"]["layers"])
if machine["storage"]["layers"] else ""))
ok("initramfs generator: %s" % (machine["boot"]["initramfs"] or "none (--yes given)"))
need = 400 << 20
if free_bytes("/boot") < need:
raise ToolError("/boot has %s free; a kernel plus initramfs needs about %s"
% (human(free_bytes("/boot")), human(need)))
ok("/boot has %s free" % human(free_bytes("/boot")))
def grubby_default(dry=False):
"""The default kernel's path, or ''. grubby exits 0 with junk when it
cannot read grubenv (unprivileged it prints a grep error, then '/boot'), and
stderr is merged in -- so only a line that names an existing FILE counts."""
p = sudo(["grubby", "--default-kernel"], check=False, capture=True, dry=dry)
for line in reversed((p.stdout or "").splitlines() if p else []):
line = line.strip()
if line.startswith("/") and os.path.isfile(line):
return line
return ""
def set_default(path, dry=False):
"""-> True only when grubby afterwards REPORTS that path as the default."""
sudo(["grubby", "--set-default", path], check=False, dry=dry)
return dry or grubby_default() == path
def install_kernel(tree, mk, machine, vendors, release, opts, state):
dry = opts["dry_run"]
say("Install preflight")
install_preflight(machine, opts, release, tree)
grubby = machine["boot"]["grubby"]
keep = SudoKeepalive()
if not dry:
keep.start()
try:
old_default = grubby_default(dry) if grubby else ""
if old_default:
info("default boot entry now: %s" % old_default)
say("Installing modules and kernel (sudo)")
if not dry:
state.update(tree, status="installing", stage="install")
# sudo resets the environment; the toolchain arguments ride on the
# command line so a clang-built tree is not re-probed with gcc.
tcfg = DotConfig(os.path.join(tree, ".config"))
strip = ["INSTALL_MOD_STRIP=1"] if tcfg.get("DEBUG_INFO_BTF") == "y" \
and state.get(tree).get("debug_info") == "btf" else []
try:
sudo(mk.argv(["-C", tree] + strip + ["modules_install"], parallel=True), dry=dry)
sudo(mk.argv(["-C", tree, "install"]), dry=dry)
finally:
# In a finally because of WHEN installkernel moves the default: before
# it runs dracut. A `make install` that dies there (a full /boot, a
# failing hook) has already pointed the machine at a kernel with no
# initramfs -- the one failure that must not also skip the restore.
if old_default and not opts["set_default"] and not dry \
and grubby_default() != old_default:
if set_default(old_default):
ok("default boot entry put back to %s" % old_default)
else:
warn("could NOT put the default boot entry back to %s -- check "
"`sudo grubby --default-kernel` before you reboot." % old_default)
# Always, overwriting: EL's installkernel does not do this, its absence
# is how the NEXT build loses its base config, and a stale one from an
# earlier install of the same release would be worse than none.
p = sudo(["install", "-m", "0644", os.path.join(tree, ".config"),
"/boot/config-" + release], check=False, dry=dry)
if p is not None and p.returncode != 0:
warn("could not write /boot/config-%s; keep %s/.config safe" % (release, tree))
if opts["set_default"]:
if not grubby:
note("no grubby here: set the default entry with your bootloader's own tool")
elif set_default("/boot/vmlinuz-" + release, dry):
ok("default boot entry is now %s, as asked" % release)
else:
warn("grubby did not accept /boot/vmlinuz-%s as the default" % release)
elif grubby:
info("pick %s from the boot menu; once it has proved itself:" % release)
info(" sudo grubby --set-default=/boot/vmlinuz-%s" % release)
else:
note("no grubby here: check which entry your bootloader now defaults to")
if not dry:
state.update(tree, status="installed", stage="done", finished=now())
post_install_checks(tree, machine, vendors, release, dry)
finally:
keep.stop()
def post_install_checks(tree, machine, vendors, release, dry=False):
say("Post-install checks")
if dry:
info("(dry run -- nothing was installed, nothing to check)")
return
img = next((i for i in ("/boot/initramfs-%s.img" % release,
"/boot/initrd.img-%s" % release) if os.path.exists(i)), "")
lister = next((t for t in ("lsinitrd", "lsinitramfs")
if which(t, path=":".join(SYSTEM_PATH))), "")
if not img:
warn("no initramfs found for %s in /boot -- do not reboot into it yet" % release)
elif lister:
p = sudo([lister, img], check=False, capture=True)
listing = p.stdout or ""
for vendor, fw in (("amd", "amdgpu"), ("intel", "i915"), ("nvidia", "nvidia")):
if vendor not in vendors:
continue
n = len(re.findall(r"firmware/%s" % fw, listing))
if n:
ok("%d %s firmware files in %s" % (n, fw, os.path.basename(img)))
elif vendor != "nvidia":
warn("no %s firmware in %s -- install linux-firmware and regenerate "
"the initramfs BEFORE rebooting" % (fw, img))
build = "/lib/modules/%s/build" % release
if os.path.exists(build):
ok("%s -> %s (DKMS can build)" % (build, os.path.realpath(build)))
else:
warn("%s is missing; DKMS cannot build against this kernel" % build)
# FILE names keep their hyphens (virtio-gpu.ko); only the module NAME uses
# underscores. Compare normalised, or 40% of modules are 'missing'.
have = set()
for dirpath, _, files in os.walk("/lib/modules/%s" % release):
for fn in files:
if ".ko" in fn:
have.add(fn.split(".ko")[0].replace("-", "_"))
cfg = DotConfig(os.path.join(tree, ".config"))
for mod, sym in (("amdgpu", "DRM_AMDGPU"), ("nouveau", "DRM_NOUVEAU"),
("i915", "DRM_I915"), ("xe", "DRM_XE"),
("simpledrm", "DRM_SIMPLEDRM"), ("virtio_gpu", "DRM_VIRTIO_GPU")):
val = cfg.get(sym)
if mod in have:
ok("%-11s installed as a module" % mod)
elif val == "y":
ok("%-11s built into the kernel image" % mod)
elif val == "m":
warn("%s: CONFIG_%s=m but no %s.ko was installed -- a real failure" % (mod, sym, mod))
if machine.get("dkms"):
p = run(["dkms", "status", "-k", release], check=False, env=scrubbed_env())
for line in (p.stdout or "").strip().splitlines():
info("dkms: " + line)
say("Done. Keep your current kernel in the boot menu until %s is proven." % release)
# ------------------------------------------------------------------ options / UI
DEFAULTS = {
"kernel": "stable", "source": "", "workdir": "~/src", "gpu": "auto",
"toolchain": "gcc", "cc": "", "llvm_prefix": "", "config_base": "auto",
"modules": "full", "cpu": "generic", "preempt": "keep", "hz": "keep",
"debug_info": "off", "localversion": "-elkt", "frame_warn": "8192",
"jobs": "0", "clean": "auto", "binutils": "auto", "linker": "default",
"python": "", "ccache": "off", "cxx": "", "secure_boot": "on",
}
PROFILE_KEYS = sorted(DEFAULTS)
CHOICES = {
"toolchain": ("gcc", "llvm"), "modules": ("full", "trimmed"),
"linker": ("default", "bfd", "lld"), "ccache": ("off", "on"), "secure_boot": ("on", "off"),
"cpu": ("generic", "native"), "preempt": ("keep",) + tuple(sorted(PREEMPT_CHOICE)),
"hz": ("keep",) + HZ_CHOICE, "debug_info": ("off", "btf", "on"),
"clean": ("auto", "always", "never", "force"),
}
PROFILE_HEADER = "# %s profile" % TOOL
PROFILE_FLAGS = ("install", "set_default") # saved too, so a reused profile asks NOTHING
def profiles_dir():
return os.path.join(os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config"),
TOOL, "profiles")
def profile_path(name, must_exist=False):
"""A bare name lives in the profiles directory; anything with a slash is a
path. (Bare names used to be written into whatever the current directory
was, where nothing ever looked for them again.)"""
name = os.path.expanduser(name)
if os.sep in name or (must_exist and os.path.isfile(name)):
return name
base = os.path.join(profiles_dir(), name)
if must_exist:
return next((c for c in (base, base + ".ini") if os.path.isfile(c)), base + ".ini")
return base if base.endswith(".ini") else base + ".ini"
def load_profile(path):
path = profile_path(path, must_exist=True)
cp = configparser.RawConfigParser()
try:
read = cp.read(path)
except configparser.Error:
read = []
if not read:
raise ToolError("cannot read profile %s" % path)
if not cp.has_section("build"):
raise ToolError("profile %s has no [build] section" % path)
return dict((k, v) for k, v in cp.items("build") if k in DEFAULTS or k in PROFILE_FLAGS)
def profile_summary(v):
cc = v.get("cc") or (v.get("llvm_prefix", "") and v["llvm_prefix"] + "clang") or \
("system clang" if v.get("toolchain") == "llvm" else "system gcc")
return "%s | %s | %s modules | cpu %s | %s%s" % (
v.get("kernel", "stable"), cc.replace(os.path.expanduser("~"), "~"),
v.get("modules", "full"), v.get("cpu", "generic"), v.get("localversion", ""),
" | installs" if v.get("install") == "yes" else "")
def list_profiles(extra_dirs=None):
"""Newest first. The profiles directory, plus -- because that is where
earlier versions put them -- the current directory and the tool's own: any
small file that starts with the profile header, whatever it is called."""
if extra_dirs is None:
extra_dirs = [os.getcwd(), os.path.dirname(os.path.abspath(__file__))]
found, seen = [], set()
for d in [profiles_dir()] + list(extra_dirs):
try:
names = sorted(os.listdir(d))[:500]
except OSError:
continue
for n in names:
path = os.path.join(d, n)
real = os.path.realpath(path)
try:
if real in seen or not os.path.isfile(path) or os.path.getsize(path) > 8192:
continue
with open(path, "r", errors="replace") as f:
if not f.readline().startswith(PROFILE_HEADER):
continue
vals = load_profile(path)
except (OSError, ToolError):
continue
seen.add(real)
found.append({"name": n[:-4] if n.endswith(".ini") else n, "path": path, "dir": d,
"mtime": os.path.getmtime(path), "values": vals,
"summary": profile_summary(vals)})
found.sort(key=lambda pr: -pr["mtime"])
return found
def choose_profile(profs):
"""The FIRST thing the wizard shows. -> {path, name, as_is} or None (new)."""
if not profs:
return None
say("Saved profiles")
for i, pr in enumerate(profs, 1):
out(" %2d) %-24s %s %s" % (i, pr["name"], time.strftime(
"%Y-%m-%d %H:%M", time.localtime(pr["mtime"])), pr["summary"]))
if pr["dir"] != profiles_dir():
out(" (in %s)" % pr["dir"])
out(" n) new -- answer the questions")
while True:
ans = ask("profile", "1")
if ans.lower() in ("n", "new"):
return None
pick = profs[int(ans) - 1] if ans.isdigit() and 1 <= int(ans) <= len(profs) else \
next((pr for pr in profs if pr["name"] == ans), None)
if pick:
break
out(" a number from the list, a profile's name, or n")
how = ask("use it as it is, or edit it? (u/e)", "u")
return {"path": pick["path"], "name": pick["name"], "as_is": not how.lower().startswith("e")}
def save_profile(path, opts):
cp = configparser.RawConfigParser()
cp.add_section("build")
for k in PROFILE_KEYS:
# What was asked, not what it resolved to HERE: a profile is for other
# machines, where -j24 and /home/<me>/src are both wrong.
cp.set("build", k, str(opts.get("_raw", {}).get(k, opts[k])))
for k in PROFILE_FLAGS:
cp.set("build", k, "yes" if opts.get(k) else "no")
path = profile_path(path)
if os.path.dirname(path):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
f.write("%s. Replay with: %s.py build --profile %s\n"
% (PROFILE_HEADER, TOOL, os.path.basename(path)[:-4]
if os.path.dirname(path) == profiles_dir() else path))
cp.write(f)
info("profile saved: %s" % path)
def validate_options(opts):
"""For values from the command line, a profile AND the wizard alike."""
for k, allowed in CHOICES.items():
if opts[k] not in allowed:
raise ToolError("%s must be one of: %s (got '%s')" % (k, " ".join(allowed), opts[k]))
lv = opts["localversion"]
if lv and lv[0] not in "-+._":
# kbuild glues it straight on: 'machine-native' gives 7.2.6machine-native.
opts["localversion"] = "-" + lv
note("LOCALVERSION '%s' has no separator -- using '-%s'" % (lv, lv))
if not re.match(r"^[-+.\w]*$", opts["localversion"]):
raise ToolError("LOCALVERSION may only contain letters, digits, - + . _ (got '%s')"
% opts["localversion"])
if not re.match(r"^\d+$", str(opts["frame_warn"])):
raise ToolError("--frame-warn must be a number")
def merge_options(args):
"""Command line beats profile beats built-in default."""
opts = dict(DEFAULTS)
prof = load_profile(args.profile) if getattr(args, "profile", None) else {}
opts.update((k, v) for k, v in prof.items() if k in DEFAULTS)
for k in DEFAULTS:
v = getattr(args, k, None)
if v is not None:
opts[k] = v
validate_options(opts)
for k in ("yes", "verbose", "require_pgp", "install", "set_default", "configure_only",
"allow_unsigned", "dry_run", "install_deps", "json"):
opts[k] = bool(getattr(args, k, False))
for k in PROFILE_FLAGS:
opts[k] = opts[k] or prof.get(k) == "yes"
opts["state_dir"] = getattr(args, "state_dir", None)
opts["save_profile"] = getattr(args, "save_profile", None)
opts["keep_symbols"] = [s.upper().replace("CONFIG_", "")
for s in (getattr(args, "keep_symbol", None) or [])]
opts["_raw"] = dict((k, opts[k]) for k in ("jobs", "workdir", "source"))
opts["workdir"] = os.path.abspath(os.path.expanduser(opts["workdir"]))
if opts["source"]:
opts["source"] = os.path.realpath(os.path.expanduser(opts["source"]))
try:
jobs = int(opts["jobs"])
except ValueError:
raise ToolError("--jobs must be a number")
opts["jobs"] = jobs if jobs > 0 else cpu_info()["threads"]
return opts
def ask(prompt, default=""):
try:
ans = input("%s [%s]: " % (prompt, default)).strip()
except EOFError:
ans = ""
return ans or default
def ask_choice(title, options, default):
"""options: list of (key, label). Accepts the number or the key."""
if default not in [k for k, _ in options]:
options = list(options) + [(default, "as given on the command line / in the profile")]
out("\n" + _c("1", title))
for i, (key, label) in enumerate(options, 1):
out(" %2d) %-12s %s%s" % (i, key, label, " <- default" if key == default else ""))
while True:
ans = ask("choose", default)
if ans.isdigit() and 1 <= int(ans) <= len(options):
return options[int(ans) - 1][0]
if ans in [k for k, _ in options]:
return ans
out(" please answer with a number or one of the keys")
def compiler_tags(t, machine, check=True):
tags = [t["origin"]]
if t["experimental"]:
tags.append("an experimental build")
if t["usable"] == "no":
return tags, False
if t["usable"] == "fork":
tags.append("NOT RECOMMENDED: " + t["why"])
if t["kind"] == "clang":
tags.append("full LLVM tool set beside it" if llvm_companions(t["path"])
else "no LLVM binutils beside it (would use GNU ld)")
elif own_binutils(t["path"]):
own = [b for b in machine.get("binutils", []) if b["dir"] == own_binutils(t["path"])]
tags.append("brings its own binutils%s" % (" " + vstr(own[0]["numver"]) if own else ""))
works = True
if check:
works, why = compiler_works(t["path"], own_binutils(t["path"]) or None)
if not works:
tags.append("BROKEN in a clean environment: " + why)
return tags, works
def show_compilers(machine, numbered=False, check=True, key="toolchains"):
"""-> the selectable ones, in the order they were numbered."""
pick = []
for t in machine.get(key, []):
tags, works = compiler_tags(t, machine, check)
label = " -"
if works and t["usable"] != "no":
pick.append(t)
label = "%3d)" % len(pick) if numbered else " "
fam = {"gcc": "g++", "clang": "clang++"}.get(t["kind"], t["kind"]) if key == "cxx" else t["kind"]
what = "%s %s" % (fam, vstr(t["numver"])) if t["usable"] != "no" \
else t["version"][:58] # say what it calls itself: "... DPC++/C++ Compiler"
out(" %s %-52s %s" % (label, t["path"], what))
out(" %s" % "; ".join(tags))
if t["usable"] == "no":
out(" cannot build a kernel: %s" % t["why"])
return pick
def wizard_toolchain(opts, machine):
out("\n" + _c("1", "Compilers found on this system"))
if not machine["toolchains"]:
# A fresh box: nothing to choose from until the dependencies are in.
out(" none yet -- keeping --toolchain %s" % opts["toolchain"])
return
pick = show_compilers(machine, numbered=True)
out(" or type the path of a compiler that is not listed")
out(" DKMS and every other out-of-tree module build runs as root with the SYSTEM")
out(" compiler. A kernel built by anything else can leave them unable to compile;")
out(" the post-build check tells you, before anything is installed.")
asked = bool(opts["cc"] or opts["llvm_prefix"] or opts["toolchain"] != DEFAULTS["toolchain"])
default = "keep" if asked else "1"
if asked:
out(" keep) what was asked for: %s" % (opts["cc"] or opts["llvm_prefix"] + "clang"
or "--toolchain " + opts["toolchain"]))
while True:
ans = ask("compiler", default)
if ans == "keep":
break
if ans.isdigit() and 1 <= int(ans) <= len(pick):
chosen = pick[int(ans) - 1]
else:
chosen = compiler_info(os.path.expanduser(ans)) if os.sep in ans else None
if not chosen or chosen["usable"] == "no":
out(" %s" % ("cannot build a kernel with that: " + chosen["why"] if chosen
else "a number from the list, or the full path of a gcc or clang"))
continue
opts["toolchain"] = "llvm" if chosen["kind"] == "clang" else "gcc"
opts["llvm_prefix"] = ""
opts["cc"] = "" if chosen["real"] == os.path.realpath("/usr/bin/gcc") else chosen["path"]
opts["binutils"] = "auto" # a new compiler: start from what goes with it
opts["cxx"] = ""
break
tc = resolve_toolchain(opts, machine)
if len(machine.get("cxx", [])) > 1:
out("\n" + _c("1", "C++ compilers found on this system"))
usable = show_compilers(machine, key="cxx")
out(" The kernel itself is C. kbuild uses a C++ compiler (HOSTCXX) for the GCC")
out(" plugins -- compiled against the chosen gcc's plugin headers, so it should be")
out(" the g++ that came with that gcc -- and for xconfig.")
keys = [("auto", "the one beside the C compiler: %s" % (tc["cxx"] or "none found"))] + \
[(x["path"], "%s %s" % (x["kind"], vstr(x["numver"]))) for x in usable]
ans = ask_choice("C++ compiler", keys, opts["cxx"] or "auto")
opts["cxx"] = "" if ans == "auto" else ans
tc = resolve_toolchain(opts, machine)
out("\n" + _c("1", "Build tools that go with %s" % tc["path"]))
for label, text in describe_tools(tc):
out(" %-10s %s" % (label, text))
if ask("change any of these? (y/n)", "n").lower().startswith("y"):
sets = [("gnu" if b["system"] else b["dir"],
"GNU binutils %s%s" % (vstr(b["numver"]), " (system)" if b["system"] else ""))
for b in machine.get("binutils", [])]
if tc["kind"] == "clang" and llvm_companions(tc["path"]):
sets.append(("llvm", "LLVM's own: ld.lld, llvm-ar, llvm-nm, llvm-objcopy ..."))
if len(sets) > 1:
opts["binutils"] = ask_choice("Binutils (assembler, linker, ar, nm, objcopy)",
sets, tc["binutils"])
if _find_lld(tc, machine):
opts["linker"] = ask_choice("Linker", [
("default", "what goes with the binutils chosen above"),
("bfd", "GNU ld.bfd"),
("lld", "LLVM's ld.lld%s" % ("" if tc["kind"] == "clang" else
" -- rarely tested with gcc-built kernels"))],
opts["linker"])
pys = machine.get("pythons", [])
if len(pys) > 1:
keys = [("default" if y["platform"] else y["path"],
"%s %s%s%s" % (y["impl"], y["version"],
" (the platform interpreter)" if y["platform"] else "",
"" if y["impl"] == "CPython" else " -- not CPython: untested with kbuild"))
for y in pys]
ans = ask_choice("Python for kbuild's scripts", keys, opts["python"] or "default")
opts["python"] = "" if ans == "default" else ans
if which("ccache", path=":".join(SYSTEM_PATH)):
opts["ccache"] = ask_choice("Compiler cache", [
("off", "every object is compiled"),
("on", "ccache: a rebuild after a clean reuses what it can")], opts["ccache"])
resolve_toolchain(opts, machine) # say so NOW if the combination cannot work
while True:
ans = ask("\nparallel make jobs", str(opts["jobs"]))
if ans.isdigit() and int(ans) > 0:
opts["jobs"] = int(ans)
break
out(" a positive number")
def wizard(opts, machine, latest, rels):
say("Kernel")
rows = [r for r in rels if r["moniker"] != "linux-next"]
for i, r in enumerate(rows, 1):
out(" %2d) %-9s %-12s %s%s%s" % (
i, r["moniker"], r["version"], r["date"], " EOL" if r["eol"] else "",
" <- default (latest stable)" if r["version"] == latest else ""))
out(" or type any version (6.12.9) or series (6.12 = its newest release)")
default = str(next((i for i, r in enumerate(rows, 1) if r["version"] == latest), 1))
if opts["kernel"] != DEFAULTS["kernel"]:
default = opts["kernel"] # --kernel / profile: Enter keeps it
ans = ask("kernel", default)
opts["kernel"] = rows[int(ans) - 1]["version"] \
if ans.isdigit() and 1 <= int(ans) <= len(rows) else ans
det = ", ".join(machine["gpu_vendors"]) or "none"
opts["gpu"] = ask_choice("GPU drivers (detected: %s)" % det, [
("auto", "what was detected; every vendor if nothing was"),
("all", "AMD + NVIDIA + Intel: one kernel for mixed hardware"),
("amd", "AMD only"), ("nvidia", "NVIDIA only (nouveau, as a module)"),
("intel", "Intel only"), ("none", "VM / BMC / rescue drivers only")], opts["gpu"])
wizard_toolchain(opts, machine)
opts["modules"] = ask_choice("Module set", [
("full", "everything the base config has -- boots on other hardware too"),
("trimmed", "only what is loaded now (%d) + GPU + rescue set: a far "
"shorter build, for THIS machine" % machine["loaded_modules"])],
opts["modules"])
opts["cpu"] = ask_choice("CPU tuning", [
("generic", "runs on any CPU of this architecture"),
("native", "-march=native: only for CPUs like this one (x86, 6.16+)")], opts["cpu"])
opts["preempt"] = ask_choice("Preemption model", [
("keep", "whatever the base config says"),
("none", "throughput: servers, batch"), ("voluntary", "the desktop/server middle"),
("full", "lowest latency: desktop, audio"),
("lazy", "full preemption, fewer needless switches (6.13+)")], opts["preempt"])
opts["hz"] = ask_choice("Timer frequency", [
("keep", "whatever the base config says"), ("100", "servers"),
("250", "balanced"), ("300", "video-friendly"), ("1000", "desktop, low latency")],
opts["hz"])
opts["debug_info"] = ask_choice("Debug info", [
("off", "no DWARF, no BTF: fastest build -- but systemd's BPF filters, "
"bpftrace and CO-RE agents will not work"),
("btf", "BTF kept, DWARF stripped at install: slower build, everything works"),
("on", "whatever the base config has, untouched: for kernel debugging")],
opts["debug_info"])
opts["secure_boot"] = ask_choice("Secure Boot", [
("on", "a Secure-Boot-ready config, and the kernel image signed with your own key; "
"a machine enrols your certificate once (mokutil) and it boots with SB enabled"),
("off", "unsigned: boots only where Secure Boot is disabled")], opts["secure_boot"])
opts["localversion"] = ask("\nLOCALVERSION suffix (release becomes e.g. 7.2.6%s)"
% opts["localversion"], opts["localversion"])
opts["install"] = ask("\nInstall after a successful build? (y/n)",
"y" if opts["install"] else "n").lower().startswith("y")
if opts["install"]:
opts["set_default"] = ask("Make it the default boot entry? (y/n)",
"y" if opts["set_default"] else "n").lower().startswith("y")
prof = ask("\nSave these answers as a profile (a name; empty = no)",
opts.get("_profile_name", ""))
if prof:
opts["save_profile"] = prof
validate_options(opts) # typed answers get the same scrutiny as flags
return opts
# ---------------------------------------------------------------------- commands
def cmd_kernels(args):
latest, rels = get_releases()
if args.series:
sums, _ = get_series_sums(int(args.series.split(".")[0]))
vers = sorted([v for v in sums if series_of(v) == args.series], key=version_key)
if args.json:
out(json.dumps(vers))
return 0
say("kernel.org: %d releases in the %s series" % (len(vers), args.series))
for i in range(0, len(vers), 8):
out(" " + " ".join("%-10s" % v for v in vers[i:i + 8]))
return 0
if args.json:
out(json.dumps({"latest_stable": latest, "releases": rels}, indent=2))
return 0
say("kernel.org right now")
for r in rels:
tag = "EOL" if r["eol"] else ""
if r["version"] == latest:
tag = "latest stable"
if r["moniker"] == "linux-next":
tag = "no tarball -- not buildable here"
out(" %-10s %-14s %-11s %s" % (r["moniker"], r["version"], r["date"], tag))
out("\n --kernel stable | longterm | mainline | 6.12 | 6.12.9"
" kernels --series 6.12 lists every point release")
return 0
def show_machine(m):
say("This machine")
info("os : %s (%s, %s)" % (m["os"], m["arch"], m["pkg_manager"] or "no known package manager"))
info("running : %s" % m["running"])
info("cpu : %s x%d %s" % (m["cpu"]["model"] or m["cpu"]["vendor"] or "?",
m["cpu"]["threads"], " ".join(m["cpu"]["simd"])))
info("memory : %s virtualisation: %s" % (human(m["memory"]), m["virt"]))
for d in m["pci"]:
info("%-12s: %s [%04x:%04x] %s%s" % (
d["kind"], d["addr"], d["vendor"], d["device"], d["name"] or
OTHER_VIDEO.get(d["vendor"], ""), " driver=" + d["driver"] if d["driver"] else ""))
if not m["pci"]:
info("display : none found")
st = m["storage"]
info("storage : root is %s; filesystems %s; layers %s"
% (st["rootfs"] or "?", ",".join(st["fstypes"]) or "-", ",".join(st["layers"]) or "none"))
info("firmware : compressed as %s" % (",".join(m["firmware_compression"]) or "not compressed / not found"))
info("secure boot : %s boot: %s%s, %s" % (
m["secure_boot"], "grubby" if m["boot"]["grubby"] else "no grubby",
"+BLS" if m["boot"]["bls"] else "", m["boot"]["initramfs"] or "no initramfs tool found"))
for t in m["toolchains"]:
info("compiler : %-46s %s %s%s" % (
t["path"], t["kind"], vstr(t["numver"]),
"" if t["system"] else " [%s]" % ("unusable" if t["usable"] == "no" else "not system")))
info("c++ : %s" % (" ".join("%s %s%s" % (x["kind"] if x["usable"] != "no" else "intel",
vstr(x["numver"]),
"" if x["usable"] != "no" else " (unusable)")
for x in m.get("cxx", [])) or "none found"))
info("binutils : %s" % (" ".join("%s in %s" % (vstr(b["numver"]), b["dir"])
for b in m.get("binutils", [])) or "none found"))
info("python3 : %s" % (" ".join("%s %s%s" % (y["impl"], y["version"],
" (platform)" if y["platform"] else "")
for y in m.get("pythons", [])) or "none found"))
info(" (`toolchains` shows all of this in full)")
for d in m["dkms"]:
lim = d["limits"].get("BUILD_EXCLUSIVE_KERNEL_MAX")
info("dkms : %s/%s on %d kernel(s)%s" % (
d["name"], d["version"], len(d["kernels"]), ", capped at %s" % lim if lim else ""))
info("modules : %d loaded" % m["loaded_modules"])
def cmd_toolchains(args):
opts = merge_options(args)
machine = detect_machine()
if opts["json"]:
out(json.dumps({"compilers": machine["toolchains"], "binutils": machine["binutils"],
"cxx": machine["cxx"], "pythons": machine["pythons"],
"tools": find_misc_tools()},
indent=2, default=list))
return 0
say("Compilers")
show_compilers(machine)
say("C++ compilers (kbuild's HOSTCXX: it builds the GCC plugins and xconfig)")
show_compilers(machine, key="cxx")
say("GNU binutils sets (a directory with both ld and as)")
for b in machine["binutils"]:
info("%-46s %s%s" % (b["dir"], b["version"], " [system]" if b["system"] else ""))
say("Python 3 interpreters")
for y in machine["pythons"]:
info("%-46s %-8s %s%s" % (y["path"], y["impl"], y["version"],
" [platform -- the default for kbuild's scripts]"
if y["platform"] else ""))
say("Other build tools")
tools = find_misc_tools()
for name, _ in MISC_TOOLS:
if name in tools:
info("%-10s %-24s %s%s" % (name, tools[name]["path"], tools[name]["version"],
" [%s]" % NOT_FOR_KERNELS[name]
if name in NOT_FOR_KERNELS else ""))
else:
info("%-10s -" % name)
say("What the current options resolve to")
tc = resolve_toolchain(opts, machine)
info("compiler : %s" % tc["version"])
info(" %s [%s]" % (tc["path"], tc["origin"]))
for label, text in describe_tools(tc):
info("%-12s: %s" % (label, text))
info("make args : %s" % (" ".join(shlex.quote(a) for a in tc["args"]) or "(none needed)"))
if tc.get("extra_path"):
info("PATH : %s:/usr/bin:..." % tc["extra_path"])
tree = opts["source"]
if not tree:
state = State(os.path.join(state_dir(opts), "state.ini"))
known = [t for t in state.trees() if os.path.isdir(t)]
tree = known[-1] if known else ""
if tree:
say("Checked against %s's own requirements" % os.path.basename(tree))
return 1 if check_tree_requirements(tree, tc, verbose=True) else 0
return 0
def cmd_profiles(args):
profs = list_profiles()
say("Saved profiles (%s)" % profiles_dir())
if not profs:
info("none yet -- the wizard offers to save one")
for pr in profs:
out(" %-24s %s %s" % (pr["name"], time.strftime("%Y-%m-%d %H:%M",
time.localtime(pr["mtime"])), pr["summary"]))
if pr["dir"] != profiles_dir():
out(" (in %s)" % pr["dir"])
out("\n build --profile <name> replays one without a single question")
return 0
def cmd_detect(args):
m = detect_machine()
if args.json:
out(json.dumps(m, indent=2, default=list))
else:
show_machine(m)
return 0
NEEDED_TOOLS = ("make", "flex", "bison", "bc", "perl", "tar", "xz", "openssl", "rsync",
"cpio", "depmod")
NEEDED_HEADERS = (("openssl/opensslv.h", "openssl-devel / libssl-dev"),
("libelf.h", "elfutils-libelf-devel / libelf-dev"))
def doctor(opts, machine, toolchain=None, quiet_ok=False):
"""-> number of blocking problems."""
say("Preflight")
problems = 0
if os.geteuid() == 0:
raise ToolError("do not run this as root. The build runs as you; only the "
"install steps use sudo.")
path = ":".join(SYSTEM_PATH)
missing = [t for t in NEEDED_TOOLS if not which(t, path=path)]
if opts.get("debug_info") in ("btf", "on") and not which("pahole", path=path):
missing.append("pahole")
cc = (toolchain or {}).get("path") or "/usr/bin/gcc"
for header, pkg in NEEDED_HEADERS:
p = run([cc, "-E", "-x", "c", "-"], stdin_data="#include <%s>\n" % header,
check=False, env=scrubbed_env()) if os.path.exists(cc) else None
if p is None or p.returncode != 0:
missing.append("%s (%s)" % (header, pkg))
if opts.get("secure_boot", "on") == "on" and not signing_tools():
missing.append("a signing tool (sbsigntools, or pesign + nss-tools)")
if missing:
problems += 1
bad("missing: " + ", ".join(missing))
pm = machine["pkg_manager"]
if pm:
info(" sudo %s %s" % (" ".join(INSTALL_CMD[pm]), BUILD_DEPS[pm]))
elif not quiet_ok:
ok("build tools and headers present")
# Measured: a full 2500-module tree without debug info is 4.7 GB built, plus
# the tarball. DWARF multiplies that several times over.
need = (4 if opts.get("modules") == "trimmed" else 10) << 30
if opts.get("debug_info") in ("btf", "on"):
need *= 4
have = free_bytes(opts["workdir"])
if have < need:
problems += 1
bad("%s has %s free; this build wants about %s" % (opts["workdir"], human(have), human(need)))
elif not quiet_ok:
ok("%s free in %s" % (human(have), opts["workdir"]))
if re.search(r"[\s:]", opts["workdir"]):
problems += 1
bad("kbuild cannot work in a path containing spaces or colons: %s" % opts["workdir"])
traps = env_traps()
if traps and not quiet_ok:
note("your shell exports things that would leak into a build; every make "
"here runs in a scrubbed environment instead:")
for t in traps:
info(" " + t)
fwdirs = {"amd": "amdgpu", "intel": "i915", "nvidia": "nvidia"}
for vendor in machine.get("gpu_vendors", []):
if not glob.glob("/lib/firmware/%s/*" % fwdirs[vendor]) and vendor != "nvidia":
warn("a %s GPU is present but /lib/firmware/%s is empty: no kernel, however "
"well configured, can start it. Install linux-firmware."
% (vendor.upper(), fwdirs[vendor]))
# Measured on kernel builds: ~1 GB per job is comfortable, DWARF doubles it.
per_job = 2 if opts.get("debug_info") in ("btf", "on") else 1
if machine["memory"] and opts.get("jobs") and \
machine["memory"] < opts["jobs"] * per_job * (1 << 30) * 0.75:
fit = max(1, int(machine["memory"] * 0.75 / (per_job << 30)))
warn("-j%d with %s of RAM: a kernel build wants about %d GB per job%s. The OOM "
"killer ends such builds as 'make was killed by signal 9'. --jobs %d fits."
% (opts["jobs"], human(machine["memory"]), per_job,
" with debug info" if per_job == 2 else "", fit))
ucode = {"GenuineIntel": "intel-ucode", "AuthenticAMD": "amd-ucode"}.get(machine["cpu"]["vendor"])
if ucode and not quiet_ok and not glob.glob("/lib/firmware/%s/*" % ucode):
note("/lib/firmware/%s is empty: no CPU microcode update will be loaded by ANY "
"kernel here (install microcode_ctl / intel-microcode / amd64-microcode, "
"then rebuild the initramfs)." % ucode)
if machine["secure_boot"] == "on":
warn("Secure Boot is ON: a kernel built here is unsigned and will not boot "
"until you sign it or turn Secure Boot off.")
return problems
def install_deps(pm, dry=False):
if not pm:
raise ToolError("no known package manager; install the build dependencies yourself")
say("Installing build dependencies (sudo %s)" % pm)
# Best-effort on purpose: one third-party repo with a bad GPG key makes dnf
# exit 1 having installed everything asked of it. Preflight is the gate.
p = sudo(INSTALL_CMD[pm] + BUILD_DEPS[pm].split(), check=False, dry=dry)
if p is not None and p.returncode != 0:
warn("%s exited %d -- continuing; preflight decides whether anything is missing"
% (pm, p.returncode))
def resolve_tree(opts, fetch):
"""-> (tree, target). With fetch=False nothing is downloaded (plan)."""
if opts["source"]:
if not os.path.isfile(os.path.join(opts["source"], "Makefile")):
raise ToolError("no kernel source at %s" % opts["source"])
return opts["source"], None
latest, rels = get_releases()
cache = {}
def sums_for(major):
if major not in cache:
cache[major] = get_series_sums(major)
return cache[major]
target = resolve_kernel(opts["kernel"], latest, rels, sums_for)
info("kernel : %s%s%s" % (target["version"],
" (%s)" % target["moniker"] if target["moniker"] else "",
" EOL -- no more fixes upstream" if target["eol"] else ""))
if fetch:
return os.path.realpath(fetch_source(target, opts["workdir"], opts["require_pgp"])), target
return os.path.join(opts["workdir"], "linux-" + target["version"]), target
def show_plan(opts, machine, toolchain, vendors, why, tree):
say("Plan")
info("tree : %s%s" % (tree, "" if os.path.isdir(tree) else " (will be downloaded)"))
info("gpu drivers : %s (%s) + VM/BMC/rescue, always" % (" ".join(sorted(vendors)) or "none", why))
info("compiler : %s" % toolchain["version"])
info(" %s [%s]" % (toolchain["path"], toolchain["origin"]))
for label, text in describe_tools(toolchain):
info("%-12s: %s" % (label, text))
if toolchain["args"]:
info("make args : %s" % " ".join(shlex.quote(a) for a in toolchain["args"]))
if toolchain["usable"] == "fork":
warn("%s -- %s." % (toolchain["path"], toolchain["why"]))
if not toolchain["system"] or toolchain["experimental"]:
warn("%s is not the system compiler%s. DKMS builds as root with the system "
"one; mixing compilers between a kernel and its modules is the usual "
"reason DKMS stops compiling." % (
toolchain["path"], " and calls itself experimental" if toolchain["experimental"] else ""))
info("modules : %s cpu: %s preempt: %s hz: %s debug info: %s"
% (opts["modules"], opts["cpu"], opts["preempt"], opts["hz"], opts["debug_info"]))
info("secure boot : %s" % ("ON -- Secure-Boot-ready config; image signed with your key (%s)"
% (keys_dir() + "/secureboot.*")
if opts["secure_boot"] == "on" else
"off -- the image will NOT boot where Secure Boot is enabled"))
info("release : <version>%s jobs: %d clean: %s"
% (opts["localversion"], opts["jobs"], opts["clean"]))
m = re.search(r"linux-(\d+\.\d+)", os.path.basename(tree))
try:
_, _, label = find_base_config(tree if os.path.isdir(tree) else "/nonexistent",
opts["config_base"],
target_series=m.group(1) if m else "",
skip_own=opts.get("_skip_own", False),
want_full=opts["modules"] == "full")
except ToolError as e:
label = "NONE -- " + str(e).splitlines()[0]
info("base config : %s" % label)
serving = in_service(tree)
if serving:
note("this tree is the DKMS build directory for installed kernel(s): %s"
% ", ".join(serving))
info("then : %s" % ("install, %s the default boot entry" % (
"and make it" if opts["set_default"] else "WITHOUT changing") if opts["install"]
else "stop after building (install later with the `install` command)"))
def cmd_plan(args):
opts = merge_options(args)
machine = detect_machine()
toolchain = resolve_toolchain(opts, machine)
vendors, why = resolve_gpu(opts["gpu"], machine["gpu_vendors"])
say("Resolving")
tree, _ = resolve_tree(opts, fetch=False)
show_plan(opts, machine, toolchain, vendors, why, tree)
problems = doctor(opts, machine, toolchain)
if os.path.isdir(tree):
problems += check_tree_requirements(tree, toolchain)
out("")
info("nothing was downloaded, written or built." + (
"" if not problems else " %d blocking problem(s) above." % problems))
return 1 if problems else 0
def cmd_doctor(args):
opts = merge_options(args)
if opts["install_deps"]:
install_deps(find_pkg_manager(), opts["dry_run"])
machine = detect_machine()
try:
toolchain = resolve_toolchain(opts, machine)
except ToolError as e:
bad(str(e))
toolchain = None
return 1 if doctor(opts, machine, toolchain) else 0
def cmd_build(args, interactive=False):
opts = merge_options(args)
if opts["install_deps"]:
# First: on a fresh box there is no compiler to detect until this ran.
install_deps(find_pkg_manager(), opts["dry_run"])
reuse = None
if interactive and not getattr(args, "profile", None):
reuse = choose_profile(list_profiles()) # FIRST: before anything else is asked
if reuse:
args.profile = reuse["path"]
opts = merge_options(args)
opts["_profile_name"] = reuse["name"]
machine = detect_machine()
if interactive and not (reuse and reuse["as_is"]):
show_machine(machine)
latest, rels = get_releases()
opts = wizard(opts, machine, latest, rels)
toolchain = resolve_toolchain(opts, machine)
vendors, why = resolve_gpu(opts["gpu"], machine["gpu_vendors"])
if doctor(opts, machine, toolchain, quiet_ok=False):
raise ToolError("preflight found blocking problems -- fix them, or re-run "
"with --install-deps to let the package manager try")
say("Source")
tree, _ = resolve_tree(opts, fetch=True)
if re.search(r"[\s:]", tree):
raise ToolError("kbuild cannot build in a path with spaces or colons: %s" % tree)
sdir = state_dir(opts)
state = State(os.path.join(sdir, "state.ini"))
prev = state.get(tree)
# A tree last configured TRIMMED must not seed a FULL build with its .config.
opts["_skip_own"] = opts["modules"] == "full" and prev.get("modules") == "trimmed"
if opts["_skip_own"]:
note("this tree's .config came from a --modules trimmed run; a full build "
"starts from a full base instead")
show_plan(opts, machine, toolchain, vendors, why, tree)
say("What this tree requires of the build tools")
if check_tree_requirements(tree, toolchain):
raise ToolError("the chosen tools do not meet linux-%s's own minimums -- see "
"`toolchains` for what else is installed" % os.path.basename(tree)[6:])
if opts["install"]:
say("Install preflight (now, not after the build)")
install_preflight(machine, opts)
if opts["save_profile"]:
save_profile(opts["save_profile"], opts)
if interactive and not ask("\nProceed? (y/n)", "y").lower().startswith("y"):
info("stopped before touching the tree.")
return 0
log_path = os.path.join(sdir, "build-%s.log" % os.path.basename(tree))
mk = Make(tree, toolchain, opts["jobs"])
say("Previous attempt on this tree")
info("recorded : %s" % ("%s%s%s" % (
prev.get("status"), " at stage " + prev["stage"] if prev.get("stage") else "",
" (%s)" % prev["finished"] if prev.get("finished") else "")
if prev else "nothing"))
has_objects = False
for dirpath, dirnames, files in os.walk(tree):
dirnames[:] = [d for d in dirnames if not (dirpath == tree and d in ("scripts", "tools", ".git"))]
if any(f.endswith(".o") for f in files):
has_objects = True
break
do_clean, why_clean = decide_clean(opts["clean"], has_objects, prev.get("status", ""),
prev.get("stage", ""), in_service(tree))
info("%-12s: %s" % ("cleaning" if do_clean else "keeping", why_clean))
keeper = None
if opts["install"] and not opts["dry_run"]:
# Before the long part, not after it.
say("sudo, once, now -- so the install at the end does not wait on a prompt")
keeper = SudoKeepalive()
keeper.start()
stage = "configure"
state.update(tree, status="configuring", stage=stage, started=now(), finished="",
exit_code="", error="", last_error="", toolchain=toolchain["version"], jobs=opts["jobs"], log=log_path)
try:
recover_held_config(tree) # before a backup could copy a staging file
if do_clean:
clean_tree(mk, tree, restore=opts["config_base"] == "auto")
if pin_toolchain(tree, toolchain, opts["secure_boot"] == "on") and pin_text(toolchain):
info("pinned : %s -- out-of-tree builds against this tree will use %s too"
% (PIN_FILE, toolchain["path"]))
say("Configuring")
staging, reqs, results, _ = configure(tree, opts, machine, mk, vendors)
say("Verifying the configuration")
fails = report_verify(results)
if fails:
rejected = os.path.join(tree, ".config.elkt-rejected")
os.replace(staging, rejected)
drop_staging_litter(staging)
raise ToolError("%d required option(s) did not come out as intended -- NOT "
"building.\n Each FAIL is a dependency that is off, not a typo: "
"`make menuconfig`, then / to search the symbol.\n The tree's "
".config was left untouched; the rejected one is %s"
% (fails, rejected))
if check_compressors(DotConfig(staging), mk.env["PATH"]):
os.replace(staging, os.path.join(tree, ".config.elkt-rejected"))
drop_staging_litter(staging)
raise ToolError("the config needs a compressor that is not installed -- NOT "
"building. Install it, or choose another CONFIG_KERNEL_* / "
"CONFIG_MODULE_COMPRESS_* with menuconfig.")
commit_config(tree, staging)
# So kernelrelease can see LOCALVERSION. Before 4.19 the target was
# called silentoldconfig.
if mk.call(["syncconfig"], check=False).returncode != 0:
mk.call(["silentoldconfig"])
predicted = mk.value("kernelrelease")
cfg = DotConfig(os.path.join(tree, ".config"))
info("config : %d built-in, %d modules" % (cfg.count("y"), cfg.count("m")))
info("release : %s" % predicted)
state.update(tree, release=predicted, config_base=opts["config_base"],
debug_info=opts["debug_info"], modules=opts["modules"])
if opts["configure_only"]:
say("Configured, not built (--configure-only)")
info("review it: make -C %s menuconfig" % tree)
info("then build: the same command without --configure-only")
state.update(tree, status="configured", stage="done", finished=now())
return 0
say("Building %s with -j%d" % (predicted, opts["jobs"]))
info("full log : %s" % log_path)
stage = "build"
state.update(tree, status="building", stage=stage)
rc, stats = run_build(mk, log_path, opts["verbose"])
if rc != 0:
out("")
bad("the build failed (%s) after %dm%02ds. First errors in the log:"
% ("make exit %d" % rc if rc > 0 else "make was killed by signal %d" % -rc,
stats["seconds"] // 60, stats["seconds"] % 60))
for h in error_summary(log_path):
out(" " + h)
raise ToolError("build failed -- full log: %s\n Re-running continues "
"incrementally from where it stopped." % log_path)
# The authority, written by the build itself.
release = read_text(os.path.join(tree, "include/config/kernel.release")).strip() or predicted
ok("built in %dm%02ds: %d steps, %d warnings" % (
stats["seconds"] // 60, stats["seconds"] % 60, stats["steps"], stats["warnings"]))
state.update(tree, status="built", stage="done", release=release,
finished=now(), seconds=stats["seconds"], warnings=stats["warnings"])
stage = "checks"
if opts["secure_boot"] == "on":
k = secure_boot_sign(tree, machine)
state.update(tree, signed="yes", certificate=k["der"])
problems = post_build_checks(tree, toolchain, machine, vendors, release)
if not opts["install"]:
say("Built, not installed.")
# The exact flags, not "the same ones": install re-resolves the
# toolchain, and a clang-built tree re-probed with gcc is a bad day.
same = "".join(" --%s %s" % (k.replace("_", "-"), shlex.quote(str(opts[k])))
for k in ("cc", "cxx", "llvm_prefix", "binutils", "linker", "python", "ccache")
if opts[k] != DEFAULTS[k])
if opts["toolchain"] != DEFAULTS["toolchain"] and not opts["cc"]:
same = " --toolchain %s%s" % (opts["toolchain"], same)
info("install with: %s install --source %s%s"
% (os.path.basename(sys.argv[0]), tree, same))
return 1 if problems else 0
if problems and not opts["yes"]:
raise ToolError("%d post-build check(s) failed -- NOT installing. The build "
"itself is intact; `install --yes` overrides." % problems)
stage = "install"
install_kernel(tree, mk, machine, vendors, release, opts, state)
return 0
except KeyboardInterrupt:
state.update(tree, status="failed", stage=stage, exit_code=130, finished=now())
raise
except Exception as e:
# 'built' must survive a failure that came after it: see decide_clean().
first = (str(e).splitlines() or [type(e).__name__])[0][:200]
if stage in ("configure", "build"):
state.update(tree, status="failed", stage=stage, finished=now(),
exit_code=1, error=first)
else:
state.update(tree, last_error="%s: %s" % (stage, first))
raise
finally:
if keeper:
keeper.stop()
def built_tree(opts, state):
if opts["source"]:
return opts["source"]
done = [(state.get(t).get("finished", ""), t) for t in state.trees()
if state.get(t).get("status") in ("built", "installing", "installed")
and os.path.isdir(t)]
if not done:
raise ToolError("no built tree on record -- pass --source <tree>")
return sorted(done)[-1][1]
def cmd_install(args):
opts = merge_options(args)
if os.geteuid() == 0:
raise ToolError("do not run this as root; it calls sudo itself where needed")
machine = detect_machine()
state = State(os.path.join(state_dir(opts), "state.ini"))
tree = built_tree(opts, state)
release = read_text(os.path.join(tree, "include/config/kernel.release")).strip()
if not release or not os.path.isfile(os.path.join(tree, "vmlinux")):
raise ToolError("%s has not been built (no vmlinux / kernel.release)" % tree)
toolchain = resolve_toolchain(opts, machine)
cfg = DotConfig(os.path.join(tree, ".config"))
vendors = set(v for v, s in (("amd", "DRM_AMDGPU"), ("nvidia", "DRM_NOUVEAU"),
("intel", "DRM_I915")) if cfg.get(s) in ("m", "y"))
info("tree : %s" % tree)
info("release : %s" % release)
mk = Make(tree, toolchain, opts["jobs"])
# `build --install` refuses to install a tree that failed these; so must this.
problems = post_build_checks(tree, toolchain, machine, vendors, release)
if problems and not opts["yes"]:
raise ToolError("%d post-build check(s) failed -- NOT installing. --yes overrides."
% problems)
install_kernel(tree, mk, machine, vendors, release, opts, state)
return 0
def cmd_sign(args):
opts = merge_options(args)
machine = detect_machine()
state = State(os.path.join(state_dir(opts), "state.ini"))
tree = built_tree(opts, state)
info("tree : %s" % tree)
k = secure_boot_sign(tree, machine)
cfg = DotConfig(os.path.join(tree, ".config"))
for sym in ("EFI_STUB", "MODULE_SIG", "MODULE_SIG_ALL"):
if cfg.get(sym) != "y":
warn("CONFIG_%s is not set in this tree: signed or not, it is not Secure-Boot-"
"ready. Rebuild with --secure-boot on." % sym)
state.update(tree, signed="yes", certificate=k["der"])
return 0
PKG_TARGET = {"rpm": "binrpm-pkg", "deb": "bindeb-pkg", "tar": "tarxz-pkg"}
INSTALL_TXT = """{release} -- built {date} with {compiler}
Configuration: {summary}
INSTALL
{steps}
SECURE BOOT
{sb}
WHAT THIS KERNEL EXPECTS OF YOUR MACHINE
{expects}
Verify the download first: sha256sum -c SHA256SUMS
"""
def tree_summary(cfg):
"""What the TREE is, read from its config -- not what this command's options
happen to say (a trimmed build packaged with default options is still trimmed)."""
preempt = next((name for name, sym in (("lazy", "PREEMPT_LAZY"), ("full", "PREEMPT"),
("voluntary", "PREEMPT_VOLUNTARY"), ("none", "PREEMPT_NONE"))
if cfg.get(sym) == "y"), "?")
gpus = [n for n, sym in (("amdgpu", "DRM_AMDGPU"), ("nouveau", "DRM_NOUVEAU"),
("i915", "DRM_I915"), ("xe", "DRM_XE")) if cfg.get(sym) in ("m", "y")]
return "%d modules, %d built-in | GPU: %s | CPU: %s | preempt %s%s, HZ %s | BTF %s" % (
cfg.count("m"), cfg.count("y"), " ".join(gpus) or "none",
"native (-march=native)" if cfg.get("X86_NATIVE_CPU") == "y" else "generic",
preempt, " (switchable at boot: preempt=)" if cfg.get("PREEMPT_DYNAMIC") == "y" else "",
cfg.get("HZ") or "?", "yes" if cfg.get("DEBUG_INFO_BTF") == "y" else "no")
def packaged_image_signed(fmt, pkg, cn):
"""Open the package and look at the vmlinuz INSIDE it."""
d = tempfile.mkdtemp(prefix="elkt-pkg-")
try:
if fmt == "rpm":
cmd = "rpm2cpio %s | cpio -id --quiet './lib/modules/*/vmlinuz' './boot/vmlinuz*'" % shlex.quote(pkg)
elif fmt == "deb":
cmd = "dpkg-deb --fsys-tarfile %s | tar -x --wildcards './boot/vmlinuz*'" % shlex.quote(pkg)
else:
cmd = "tar -xf %s --wildcards '*/vmlinuz*' 'boot/vmlinuz*'" % shlex.quote(pkg)
run(["sh", "-c", cmd], cwd=d, check=False, env=scrubbed_env())
for dirpath, _, files in os.walk(d):
for fn in files:
if fn.startswith("vmlinuz"):
return cn in image_signers(os.path.join(dirpath, fn))
return None
finally:
shutil.rmtree(d, ignore_errors=True)
def cmd_package(args):
opts = merge_options(args)
machine = detect_machine()
state = State(os.path.join(state_dir(opts), "state.ini"))
tree = built_tree(opts, state)
toolchain = resolve_toolchain(opts, machine)
cfg = DotConfig(os.path.join(tree, ".config"))
if (cfg.get("CC_VERSION_TEXT") or toolchain["version"]) != toolchain["version"]:
raise ToolError("%s was built with\n %s\n not with the compiler given here. Pass "
"the same --cc (packaging runs make again)." % (tree, cfg.get("CC_VERSION_TEXT")))
fmt = getattr(args, "format", None) or (
"rpm" if which("rpmbuild", path=":".join(SYSTEM_PATH)) else
"deb" if which("dpkg-deb", path=":".join(SYSTEM_PATH)) else "tar")
if fmt not in PKG_TARGET:
raise ToolError("--format must be rpm, deb or tar")
need = {"rpm": "rpmbuild", "deb": "dpkg-deb", "tar": "tar"}[fmt]
if not which(need, path=":".join(SYSTEM_PATH)):
raise ToolError("--format %s needs %s, which is not installed" % (fmt, need))
sb = opts["secure_boot"] == "on"
k = signing_key() if sb else None
say("Packaging %s as %s" % (tree, fmt))
pin_toolchain(tree, toolchain, sb) # the wrapper is what keeps the image signed
log_path = os.path.join(state_dir(opts), "package-%s.log" % os.path.basename(tree))
info("full log : %s" % log_path)
mk = Make(tree, toolchain, opts["jobs"])
started = time.time()
# NOT ELKT_NO_SIGN here: the package's own build step relinks the kernel, and
# the wrapper re-signing at the end of it is the whole point.
rc, stats = run_build(mk, log_path, opts["verbose"], [PKG_TARGET[fmt]],
{"ELKT_NO_SIGN": "" if sb else "1"})
if rc != 0:
bad("packaging failed (make exit %d). First errors in the log:" % rc)
for h in error_summary(log_path):
out(" " + h)
raise ToolError("see %s" % log_path)
release = read_text(os.path.join(tree, "include/config/kernel.release")).strip()
where = {"rpm": os.path.join(tree, "rpmbuild", "RPMS", "**", "*.rpm"),
"deb": os.path.join(os.path.dirname(tree), "*.deb"),
"tar": os.path.join(tree, "*.tar.xz")}[fmt]
pkgs = sorted(f for f in glob.glob(where, recursive=True) if os.path.getmtime(f) >= started - 5)
if not pkgs:
raise ToolError("make succeeded but no %s package newer than this run was found (%s)" % (fmt, where))
outdir = os.path.abspath(os.path.expanduser(getattr(args, "output", None) or
os.path.join("~", "kernel-packages", release)))
os.makedirs(outdir, exist_ok=True)
for f in pkgs:
shutil.copy2(f, outdir)
ok("packaged in %dm%02ds" % (stats["seconds"] // 60, stats["seconds"] % 60))
main_pkg = next((f for f in pkgs if re.search(r"(kernel|linux-image)-?\d", os.path.basename(f))
and "devel" not in f and "headers" not in f and "dbg" not in f), pkgs[0])
if sb:
signed = packaged_image_signed(fmt, main_pkg, key_cn(k))
if signed:
ok("the kernel image INSIDE %s is signed with '%s'" % (os.path.basename(main_pkg), key_cn(k)))
else:
raise ToolError("the kernel image inside %s is NOT signed%s -- not publishing it.\n "
"(The tree's %s signs after the package's build step; was it removed?)"
% (os.path.basename(main_pkg), "" if signed is False else " (could not look)",
PIN_FILE))
shutil.copy2(k["der"], os.path.join(outdir, "secureboot.der"))
has_devel = any("devel" in os.path.basename(f) or "headers" in os.path.basename(f) for f in pkgs)
names = [os.path.basename(f) for f in pkgs]
steps = {"rpm": " sudo dnf install ./%s\n reboot, and pick %s from the boot menu" % (
" ./".join(n for n in names if "headers" not in n), release),
"deb": " sudo apt install ./%s\n reboot, and pick %s from the boot menu" % (" ./".join(names), release),
"tar": " sudo tar -C / -xf %s\n sudo depmod %s && sudo dracut --kver %s (or update-initramfs -c -k %s)\n"
" add a boot entry (grubby --add-kernel / update-grub), reboot" % (names[0], release, release, release)}[fmt]
native = cfg.get("X86_NATIVE_CPU") == "y"
expects = [" CPU: %s" % ("built with -march=native on '%s' -- it needs a CPU with at least that "
"instruction set, and will not boot on an older one" % machine["cpu"]["model"]
if native else "any %s CPU" % machine["arch"]),
" Out-of-tree modules (DKMS, NVIDIA, VirtualBox): %s" % (
"install the kernel-devel package too; they are built with YOUR system compiler"
if has_devel else "NOT possible -- no kernel-devel package was produced (kbuild omits "
"it when CC and HOSTCC differ)")]
if not toolchain["system"]:
expects.append(" ! built with %s, which is not a distro compiler: module builds on another "
"machine need a compiler that accepts the same flags. For a kernel other "
"people install, build with the system compiler." % toolchain["version"][:60])
sbtxt = (" The image is signed with '%s'. With Secure Boot ENABLED, trust it once:\n"
" sudo mokutil --import secureboot.der (pick a one-time password)\n"
" reboot -> blue MOK Manager screen -> Enroll MOK -> that password\n"
" With Secure Boot disabled, skip this." % key_cn(k)) if sb else \
" NOT signed: boots only with Secure Boot disabled."
with open(os.path.join(outdir, "INSTALL.txt"), "w") as f:
f.write(INSTALL_TXT.format(release=release, date=time.strftime("%Y-%m-%d"),
compiler=toolchain["version"][:70],
summary=tree_summary(cfg),
steps=steps, sb=sbtxt, expects="\n".join(expects)))
with open(os.path.join(outdir, "SHA256SUMS"), "w") as f:
for n in sorted(os.listdir(outdir)):
if n != "SHA256SUMS":
f.write("%s %s\n" % (sha256_file(os.path.join(outdir, n)), n))
say("Ready to publish: %s" % outdir)
for n in sorted(os.listdir(outdir)):
info("%10s %s" % (human(os.path.getsize(os.path.join(outdir, n))), n))
if not toolchain["system"]:
warn("built with a non-distro compiler -- see INSTALL.txt, 'WHAT THIS KERNEL EXPECTS'")
state.update(tree, packaged=outdir)
return 0
def cmd_pin(args):
opts = merge_options(args)
machine = detect_machine()
state = State(os.path.join(state_dir(opts), "state.ini"))
tree = built_tree(opts, state)
toolchain = resolve_toolchain(opts, machine)
cfg = DotConfig(os.path.join(tree, ".config"))
built_by = cfg.get("CC_VERSION_TEXT") or ""
info("tree : %s" % tree)
info("built with : %s" % (built_by or "unknown -- not configured?"))
if built_by and built_by != toolchain["version"]:
raise ToolError("that is not the compiler given here (%s).\n A pin must name the "
"compiler the tree was BUILT with: pass the same --cc."
% toolchain["version"])
if pin_toolchain(tree, toolchain, opts["secure_boot"] == "on") and pin_text(toolchain):
ok("pinned: %s" % os.path.join(tree, PIN_FILE))
else:
info("nothing to pin: this is the system toolchain, which is what everything uses anyway")
release = read_text(os.path.join(tree, "include/config/kernel.release")).strip()
return 0 if dkms_check(tree, toolchain, machine, release) else 1
def cmd_dkms_check(args):
opts = merge_options(args)
machine = detect_machine()
state = State(os.path.join(state_dir(opts), "state.ini"))
tree = built_tree(opts, state)
toolchain = resolve_toolchain(opts, machine)
release = read_text(os.path.join(tree, "include/config/kernel.release")).strip()
info("tree : %s (%s)" % (tree, release or "release unknown -- not built?"))
return 0 if dkms_check(tree, toolchain, machine, release) else 1
def cmd_status(args):
opts = merge_options(args)
state = State(os.path.join(state_dir(opts), "state.ini"))
say("Known source trees (%s)" % state.path)
if not state.trees():
info("none yet")
for tree in state.trees():
s = state.get(tree)
gone = "" if os.path.isdir(tree) else " [directory is gone]"
out("\n %s%s" % (tree, gone))
info(" %s%s release %s %s" % (
s.get("status", "?"), " at " + s["stage"] if s.get("stage") not in ("", "done", None) else "",
s.get("release", "?"), s.get("finished") or "started " + s.get("started", "?")))
serving = in_service(tree)
if serving:
info(" in service as the DKMS build dir of: %s" % ", ".join(serving))
if s.get("error") or s.get("last_error"):
info(" last error: %s" % (s.get("error") or s.get("last_error")))
return 0
def build_parser():
common = argparse.ArgumentParser(add_help=False)
common.add_argument("--state-dir", help="where state.ini and build logs live "
"(default: ~/.local/state/%s)" % TOOL)
common.add_argument("--json", action="store_true", help="machine-readable output")
b = argparse.ArgumentParser(add_help=False)
b.add_argument("--kernel", help="stable | longterm | mainline | 6.12 | 6.12.9 (default stable)")
b.add_argument("--source", help="an existing kernel tree; skips the download")
b.add_argument("--workdir", help="where trees are downloaded and built (default ~/src)")
b.add_argument("--gpu", help="auto | all | none | amd,nvidia,intel (default auto)")
b.add_argument("--toolchain", help="gcc | llvm (default gcc = the SYSTEM gcc)")
b.add_argument("--cc", help="the compiler: any gcc or clang, by path or name "
"(see the `toolchains` command)")
b.add_argument("--llvm-prefix", dest="llvm_prefix", help="directory holding clang, ld.lld ...")
b.add_argument("--cxx", help="the C++ compiler (kbuild's HOSTCXX: GCC plugins, xconfig); "
"default: the one installed beside --cc")
b.add_argument("--binutils", help="auto | gnu | llvm | <dir holding ld and as> (default auto: "
"what goes with the compiler)")
b.add_argument("--linker", help="default | bfd | lld")
b.add_argument("--python", help="the python3 kbuild's scripts run under (default /usr/bin/python3)")
b.add_argument("--ccache", help="off | on")
b.add_argument("--config-base", dest="config_base",
help="auto | running | defconfig | <file> (default auto)")
b.add_argument("--modules", help="full | trimmed (localmodconfig + GPU + rescue set)")
b.add_argument("--keep-symbol", action="append",
help="with --modules trimmed: also keep this CONFIG symbol (repeatable)")
b.add_argument("--cpu", help="generic | native")
b.add_argument("--preempt", help="keep | none | voluntary | full | lazy")
b.add_argument("--hz", help="keep | 100 | 250 | 300 | 1000")
b.add_argument("--debug-info", dest="debug_info", help="off | btf | on")
b.add_argument("--secure-boot", dest="secure_boot",
help="on | off (default on: Secure-Boot-ready config, and the image signed "
"with your key -- see the `sign` command)")
b.add_argument("--localversion", help="release suffix (default -elkt)")
b.add_argument("--frame-warn", dest="frame_warn", help="-Wframe-larger-than (default 8192)")
b.add_argument("--jobs", help="parallel jobs (default: all CPUs)")
b.add_argument("--clean", help="auto | always | never | force")
b.add_argument("--profile", help="a saved profile, by name (see `profiles`) or by path")
b.add_argument("--save-profile", dest="save_profile", help="write the resolved answers to a profile")
b.add_argument("--require-pgp", action="store_true", help="refuse an unverified sha256sums.asc")
b.add_argument("--install-deps", action="store_true", help="run the package manager first (sudo)")
b.add_argument("--configure-only", action="store_true",
help="stop once the config is verified and written; build nothing")
b.add_argument("--install", action="store_true", help="install after a successful build")
b.add_argument("--set-default", action="store_true", help="make the new kernel the default boot entry")
b.add_argument("--allow-unsigned", action="store_true", help="install even though Secure Boot is on")
b.add_argument("--dry-run", action="store_true", help="install: print the sudo commands, run none")
b.add_argument("--yes", action="store_true", help="do not stop for confirmations")
b.add_argument("--verbose", action="store_true", help="stream the whole build instead of a status line")
p = argparse.ArgumentParser(
prog=TOOL + ".py", formatter_class=argparse.RawDescriptionHelpFormatter,
description="Pick, configure, build and install a Linux kernel for this machine.\n"
"With no command on a terminal it asks; every question is also a flag.")
p.add_argument("--version", action="version", version=VERSION)
sub = p.add_subparsers(dest="command")
k = sub.add_parser("kernels", parents=[common], help="what kernel.org offers right now")
k.add_argument("--series", help="list every point release of a series, e.g. 6.12")
sub.add_parser("detect", parents=[common], help="what this machine is")
sub.add_parser("profiles", parents=[common], help="the saved profiles")
sub.add_parser("toolchains", parents=[common, b],
help="every compiler, binutils set and python here; what would be used")
sub.add_parser("doctor", parents=[common, b], help="can this machine build a kernel?")
sub.add_parser("plan", parents=[common, b], help="resolve everything, touch nothing")
sub.add_parser("build", parents=[common, b], help="fetch, configure, verify, build, check")
sub.add_parser("wizard", parents=[common, b], help="ask, then build")
sub.add_parser("install", parents=[common, b], help="install a built tree")
sub.add_parser("dkms-check", parents=[common, b], help="will out-of-tree modules compile?")
pk = sub.add_parser("package", parents=[common, b],
help="turn a built tree into an RPM / deb / tarball other people can install")
pk.add_argument("--format", help="rpm | deb | tar (default: what this machine can build)")
pk.add_argument("--output", help="where the finished set goes (default ~/kernel-packages/<release>)")
sub.add_parser("pin", parents=[common, b],
help="make a built tree use ITS compiler for every out-of-tree build (DKMS, installers)")
sub.add_parser("sign", parents=[common, b],
help="sign a built tree's kernel image for Secure Boot (makes your key on first use)")
sub.add_parser("status", parents=[common, b], help="recorded state of every known tree")
return p
def _as_interrupt(signum, frame):
raise KeyboardInterrupt()
COMMANDS = ("kernels", "detect", "toolchains", "profiles", "sign", "pin", "package", "doctor", "plan", "build", "wizard", "install",
"dkms-check", "status")
def normalize_argv(argv):
argv = list(argv)
# `tool.py --kernel 6.12` means "ask me the rest". argparse cannot express a
# default subcommand, so put it there before it looks. Only argv[0] can be
# the command: a VALUE may well be spelled 'build' (--workdir build).
if not argv or argv[0] not in COMMANDS + ("-h", "--help", "--version"):
argv = ["wizard"] + argv
# Every LOCALVERSION starts with '-', which argparse reads as an option and
# answers with "expected one argument". Glue the pair into --opt=value.
for i in range(len(argv) - 1):
if argv[i] == "--localversion":
argv[i:i + 2] = ["--localversion=" + argv[i + 1]]
break
return argv
def main(argv=None):
parser = build_parser()
argv = normalize_argv(sys.argv[1:] if argv is None else argv)
# A dropped ssh session is a SIGHUP, systemd stopping a unit a SIGTERM.
# Unhandled, neither runs a finally: state stays 'building' and the next run
# cleans a good tree. As KeyboardInterrupt they take the ^C path.
for sig in (signal.SIGTERM, signal.SIGHUP):
signal.signal(sig, _as_interrupt)
args = parser.parse_args(argv)
try:
if args.command == "wizard":
if not sys.stdin.isatty():
out("Not a terminal: nothing to ask. Use `build` with flags or "
"--profile, or see --help.")
return 2
return cmd_build(args, interactive=True)
return {"kernels": cmd_kernels, "detect": cmd_detect, "doctor": cmd_doctor,
"plan": cmd_plan, "build": cmd_build, "install": cmd_install,
"dkms-check": cmd_dkms_check, "status": cmd_status,
"toolchains": cmd_toolchains, "profiles": cmd_profiles,
"sign": cmd_sign, "pin": cmd_pin, "package": cmd_package}[args.command](args)
except ToolError as e:
out("\n" + _c("31", "ERROR:") + " " + str(e))
return 1
except KeyboardInterrupt:
out("\ninterrupted -- the tree is consistent; re-running continues incrementally.")
return 130
def _ensure_platform_python():
"""Started as `python3 tool.py` where python3 is PyPy or a private build?
Re-exec under the platform interpreter, once."""
if sys.implementation.name == "cpython" or os.environ.get("ELKT_NO_REEXEC"):
return
if os.path.exists(PLATFORM_PYTHON) and \
os.path.realpath(sys.executable) != os.path.realpath(PLATFORM_PYTHON):
os.environ["ELKT_NO_REEXEC"] = "1"
os.execv(PLATFORM_PYTHON, [PLATFORM_PYTHON, "-I", os.path.abspath(__file__)] + sys.argv[1:])
if __name__ == "__main__":
_ensure_platform_python()
sys.exit(main())Text Download
please support me by leaving a comment!