linux: rewrite updater-script, make data-drivenorigin/kernels-refactoring

13 files changed, 157 insertions, 219 deletions

diff --git a/pkgs/os-specific/linux/kernel/linux-4.14.nix b/pkgs/os-specific/linux/kernel/linux-4.14.nix
deleted file mode 100644
index dc0ead44be43..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-4.14.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "4.14.325";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v4.x/linux-${version}.tar.xz";
-    sha256 = "117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-4.19.nix b/pkgs/os-specific/linux/kernel/linux-4.19.nix
deleted file mode 100644
index 6048caf8fa86..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-4.19.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "4.19.294";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v4.x/linux-${version}.tar.xz";
-    sha256 = "03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-5.10.nix b/pkgs/os-specific/linux/kernel/linux-5.10.nix
deleted file mode 100644
index 213bf7e670f3..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-5.10.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "5.10.195";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-    sha256 = "0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-5.15.nix b/pkgs/os-specific/linux/kernel/linux-5.15.nix
deleted file mode 100644
index d86b0cf0ce64..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-5.15.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "5.15.132";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-    sha256 = "1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/linux-5.4.nix b/pkgs/os-specific/linux/kernel/linux-5.4.nix
deleted file mode 100644
index 0013636076a0..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-5.4.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "5.4.256";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-    sha256 = "0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-6.1.nix b/pkgs/os-specific/linux/kernel/linux-6.1.nix
deleted file mode 100644
index df99c9868510..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-6.1.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "6.1.54";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
-    sha256 = "09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/linux-6.4.nix b/pkgs/os-specific/linux/kernel/linux-6.4.nix
deleted file mode 100644
index d41cd21fbef9..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-6.4.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, fetchurl, buildLinux, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "6.4.16";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
-    sha256 = "0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/linux-6.5.nix b/pkgs/os-specific/linux/kernel/linux-6.5.nix
deleted file mode 100644
index 28557090de61..000000000000
--- a/pkgs/os-specific/linux/kernel/linux-6.5.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, fetchurl, buildLinux, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "6.5.4";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
-    sha256 = "0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/mainline-kernels.json b/pkgs/os-specific/linux/kernel/mainline-kernels.json
new file mode 100644
index 000000000000..3019adc4b00f
--- /dev/null
+++ b/pkgs/os-specific/linux/kernel/mainline-kernels.json
@@ -0,0 +1,34 @@
+{
+    "6.5": {
+        "version": "6.5.4",
+        "hash": "0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx"
+    },
+    "6.4": {
+        "version": "6.4.16",
+        "hash": "0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln"
+    },
+    "6.1": {
+        "version": "6.1.54",
+        "hash": "09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653"
+    },
+    "5.15": {
+        "version": "5.15.132",
+        "hash": "1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1"
+    },
+    "5.10": {
+        "version": "5.10.195",
+        "hash": "0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1"
+    },
+    "5.4": {
+        "version": "5.4.256",
+        "hash": "0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967"
+    },
+    "4.19": {
+        "version": "4.19.294",
+        "hash": "03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc"
+    },
+    "4.14": {
+        "version": "4.14.325",
+        "hash": "117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav"
+    }
+}
diff --git a/pkgs/os-specific/linux/kernel/mainline.nix b/pkgs/os-specific/linux/kernel/mainline.nix
new file mode 100644
index 000000000000..96ee5203b5e1
--- /dev/null
+++ b/pkgs/os-specific/linux/kernel/mainline.nix
@@ -0,0 +1,18 @@
+{ branch, lib, fetchurl, buildLinux, ... } @ args:
+
+let
+  allKernels = builtins.fromJSON (builtins.readFile ./mainline-kernels.json);
+  thisKernel = allKernels.${branch};
+
+  args' = (builtins.removeAttrs args ["branch"]) // rec {
+    inherit (thisKernel) version;
+    modDirVersion = lib.versions.pad 3 version;
+    extraMeta.branch = branch;
+
+    src = fetchurl {
+      url = "mirror://kernel/linux/kernel/v${lib.versions.major version}.x/linux-${version}.tar.xz";
+      sha256 = thisKernel.hash;
+    };
+  } // (args.argsOverride or {});
+in
+buildLinux args'
diff --git a/pkgs/os-specific/linux/kernel/update-mainline.py b/pkgs/os-specific/linux/kernel/update-mainline.py
new file mode 100755
index 000000000000..aeaea53510ca
--- /dev/null
+++ b/pkgs/os-specific/linux/kernel/update-mainline.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+import html.parser
+import json
+import pathlib
+import re
+import subprocess
+import urllib.request
+
+
+HERE = pathlib.Path(__file__).parent
+ROOT = HERE.parent.parent.parent.parent
+VERSIONS_FILE = HERE / "mainline-kernels.json"
+
+
+class LinksParser(html.parser.HTMLParser):
+    def __init__(self, *, convert_charrefs: bool = True) -> None:
+        super().__init__(convert_charrefs=convert_charrefs)
+        self.kernels = {}
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attrs_ = dict(attrs)
+        if tag == "a" and attrs_.get("title") == "Download complete tarball":
+            url = attrs_["href"]
+            assert url
+
+            if "-rc" in url:
+                return
+
+            version = re.search("linux-(.*).tar.xz", url)
+            assert version
+
+            self.kernels[version.group(1)] = url
+
+
+def get_branch(version: str):
+    major, minor, *_ = version.split(".")
+    return f"{major}.{minor}"
+
+
+def get_hash(url: str):
+    return subprocess.check_output(["nix-prefetch-url", url]).decode().strip()
+
+
+def commit(message):
+    return subprocess.check_call(["git", "commit", "-m", message, VERSIONS_FILE])
+
+
+def main():
+    kernel_org = urllib.request.urlopen("https://kernel.org/")
+    parser = LinksParser()
+    parser.feed(kernel_org.read().decode())
+
+    all_kernels = json.load(VERSIONS_FILE.open())
+
+    for version, link in parser.kernels.items():
+        branch = get_branch(version)
+
+        old_version = all_kernels.get(branch, {}).get("version")
+        if old_version == version:
+            print(f"linux-{branch}: {version} is latest, skipping...")
+            continue
+
+        if old_version is None:
+            message = f"linux-{branch}: init at {version}"
+        else:
+            message = f"linux-{branch}: {old_version} -> {version}"
+
+        print(message)
+
+        all_kernels[branch] = {"version": version, "hash": get_hash(link)}
+
+        with VERSIONS_FILE.open("w") as fd:
+            json.dump(all_kernels, fd, indent=4)
+            fd.write("\n")  # makes editorconfig happy
+
+        commit(message)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/pkgs/os-specific/linux/kernel/update.sh b/pkgs/os-specific/linux/kernel/update.sh
index 4171b7492b7b..37e1cc1a5cd4 100755
--- a/pkgs/os-specific/linux/kernel/update.sh
+++ b/pkgs/os-specific/linux/kernel/update.sh
@@ -1,72 +1,14 @@
 #!/usr/bin/env bash
-set -e
+cd "$(dirname "$(readlink -f "$0")")" || exit
 
-# Get the latest versions from kernel.org
-LINUXSED='s/.*linux-\([0-9]\+\(.[0-9]\+\)*\).*/\1/p'
-KDATA="$(curl -s https://www.kernel.org | sed -n -e '/Download complete/p')"
-VERSIONS=($(sed -n -e $LINUXSED <<< "$KDATA" | sort -Vr))
+echo "Update linux (mainline)"
+COMMIT=1 ./update-mainline.py || echo "update-mainline failed with exit code $?"
 
-# Remove mainline version if there is a stable update
-# Note due to sorting these two will always exist at the bottom
-if grep -q "^${VERSIONS[1]}" <<< "${VERSIONS[0]}"; then
-  VERSIONS=(${VERSIONS[@]:0:1} ${VERSIONS[@]:2})
-fi
+echo "Update linux-rt"
+COMMIT=1 ./update-rt.sh || echo "update-rt failed with exit code $?"
 
-# Inspect each file and see if it has the latest version
-NIXPKGS="$(git rev-parse --show-toplevel)"
-ls $NIXPKGS/pkgs/os-specific/linux/kernel | while read FILE; do
-  KERNEL="$(sed -n -e $LINUXSED <<< "$FILE")"
-  [ -z "$KERNEL" ] && continue
+echo "Update linux-libre"
+COMMIT=1 ./update-libre.sh || echo "update-libre failed with exit code $?"
 
-  # Find the matching new kernel version
-  MATCHING=""
-  for V in "${VERSIONS[@]}"; do
-    if grep -q "^$KERNEL" <<< "$V"; then
-      MATCHING="$V"
-      break
-    fi
-  done
-  if [ -z "$MATCHING" ]; then
-    echo "Out-of-support $KERNEL"
-    continue
-  fi
-
-  # Inspect the nix expression to check for changes
-  DATA="$(<$NIXPKGS/pkgs/os-specific/linux/kernel/$FILE)"
-  URL="$(sed -n -e 's/.*url = "\(.*\)";.*/\1/p' <<< "$DATA" | sed -e "s/\${version}/$MATCHING/g")"
-  OLDVER=$(sed -n -e 's/.*version = "\(.*\)".*/\1/p' <<< "$DATA")
-  if [ "$OLDVER" = "$V" ]; then
-    echo "No updates for $KERNEL"
-    continue
-  fi
-
-  # Download the new file for the hash
-  if ! HASH="$(nix-prefetch-url $URL 2>/dev/null)"; then
-    echo "Failed to get hash of $URL"
-    continue
-  fi
-  sed -i -e "s/sha256 = \".*\"/sha256 = \"$HASH\"/g" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-
-  # Rewrite the expression
-  sed -i -e '/version = /d' $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-  sed -i -e "\#buildLinux (args // rec {#a \  version = \"$V\";" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-
-  # Commit the changes
-  git add -u $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-  git commit -m "linux: $OLDVER -> $V" >/dev/null 2>&1
-
-  echo "Updated $OLDVER -> $V"
-done
-
-# Allowing errors again: one broken update script shouldn't inhibit the
-# update of other kernel variants.
-set +e
-
-echo Update linux-rt
-COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-rt.sh || echo "update-rt failed with exit code $?"
-
-echo Update linux-libre
-COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-libre.sh || echo "update-libre failed with exit code $?"
-
-echo Update linux-hardened
-COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/hardened/update.py || echo "update-hardened failed with exit code $?"
+echo "Update linux-hardened"
+COMMIT=1 ./hardened/update.py || echo "update-hardened failed with exit code $?"
diff --git a/pkgs/top-level/linux-kernels.nix b/pkgs/top-level/linux-kernels.nix
index 39336f07529b..e62dc63d0113 100644
--- a/pkgs/top-level/linux-kernels.nix
+++ b/pkgs/top-level/linux-kernels.nix
@@ -96,7 +96,8 @@ in {
       rpiVersion = 4;
     };
 
-    linux_4_14 = callPackage ../os-specific/linux/kernel/linux-4.14.nix {
+    linux_4_14 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "4.14";
       kernelPatches =
         [ kernelPatches.bridge_stp_helper
           kernelPatches.request_key_helper
@@ -107,7 +108,8 @@ in {
         ];
     };
 
-    linux_4_19 = callPackage ../os-specific/linux/kernel/linux-4.19.nix {
+    linux_4_19 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "4.19";
       kernelPatches =
         [ kernelPatches.bridge_stp_helper
           kernelPatches.request_key_helper
@@ -115,7 +117,8 @@ in {
         ];
     };
 
-    linux_5_4 = callPackage ../os-specific/linux/kernel/linux-5.4.nix {
+    linux_5_4 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "5.4";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -130,7 +133,8 @@ in {
       ];
     };
 
-    linux_5_10 = callPackage ../os-specific/linux/kernel/linux-5.10.nix {
+    linux_5_10 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "5.10";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -145,7 +149,8 @@ in {
       ];
     };
 
-    linux_5_15 = callPackage ../os-specific/linux/kernel/linux-5.15.nix {
+    linux_5_15 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "5.15";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -160,7 +165,8 @@ in {
       ];
     };
 
-    linux_6_1 = callPackage ../os-specific/linux/kernel/linux-6.1.nix {
+    linux_6_1 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "6.1";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -175,14 +181,16 @@ in {
       ];
     };
 
-    linux_6_4 = callPackage ../os-specific/linux/kernel/linux-6.4.nix {
+    linux_6_4 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "6.4";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
       ];
     };
 
-    linux_6_5 = callPackage ../os-specific/linux/kernel/linux-6.5.nix {
+    linux_6_5 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "6.5";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper