maintainers/scripts/update-ruby-packages: Add version regression checks

2 files changed, 264 insertions, 12 deletions

diff --git a/maintainers/scripts/update-ruby-packages b/maintainers/scripts/update-ruby-packages
index 11e6144bbb9d..b1de427029cd 100755
--- a/maintainers/scripts/update-ruby-packages
+++ b/maintainers/scripts/update-ruby-packages
@@ -1,17 +1,59 @@
 #!/usr/bin/env nix-shell
 #!nix-shell -i bash -p bundler bundix nixfmt
+# shellcheck shell=bash
 
 set -euf -o pipefail
 
-(
-  cd pkgs/development/ruby-modules/with-packages
-  rm -f gemset.nix Gemfile.lock
-  # Since bundler 2+, the lock command generates a platform-dependent
-  # Gemfile.lock, hence causing to bundix to generate a gemset tied to the
-  # platform from where it was executed.
-  BUNDLE_FORCE_RUBY_PLATFORM=1 bundle lock
-  bundix
-  nixfmt gemset.nix
-  mv gemset.nix ../../../top-level/ruby-packages.nix
-  rm -f Gemfile.lock
-)
+self="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
+
+getSpecifiedGems() {
+  grep '^\s*gem' "$1" | cut -d"'" -f2
+}
+
+cd pkgs/development/ruby-modules/with-packages
+
+# Cleanup possible leftovers from a failed run.
+rm -f gemset.nix Gemfile.lock
+
+# Since bundler 2+, the lock command generates a platform-dependent
+# Gemfile.lock, hence causing to bundix to generate a gemset tied to the
+# platform from where it was executed.
+BUNDLE_FORCE_RUBY_PLATFORM=1 bundle lock
+bundix
+nixfmt gemset.nix
+
+# Run checks against the update.
+if ! \
+  nix-instantiate --eval --strict \
+    --argstr specifiedGems "$(getSpecifiedGems Gemfile)"\
+    --arg old ../../../top-level/ruby-packages.nix \
+    --arg new ./gemset.nix \
+    "$self/update-ruby-packages.checks.nix"
+then
+  (
+  echo ""
+  echo "NOTE: The Gemfile.lock and gemset.nix files were left intact for comparison."
+  echo ""
+  echo "Do not simply continue through with the update."
+  echo "Make sure to get the Ruby maintainers involved in finding a solution to this problem."
+  echo ""
+  echo "The non-specified gems listed are generally not at fault."
+  echo "Regressions likely come from specified gems getting updated and having clashing requirements."
+  echo ""
+  echo "Start by pessimistically pinning (~>) specified gems to their current full version that look like they could be the cause."
+  echo "Then once only non-specified gems are regressed, pessimistically pin the leftover ones."
+  echo "Once this passes with pessimistic pinning of gems, try reducing specificity in pessimistic bounds, then try using minimum version bounds (>=)."
+  echo "At some point bundler will tell you why it can't give you the bounds being asked for."
+  echo ""
+  echo "Don't forget to re-generate the ruby-packages.nix nix from scratch for the proper report once the minimum required set of pins is known!"
+  echo ""
+  ) >&2
+  exit 1
+fi
+
+{
+  echo "# This file is generated and should be updated with maintainers/scripts/update-ruby-packages."
+  echo ""
+  cat gemset.nix
+} > ../../../top-level/ruby-packages.nix
+rm -v -f gemset.nix Gemfile.lock
diff --git a/maintainers/scripts/update-ruby-packages.checks.nix b/maintainers/scripts/update-ruby-packages.checks.nix
new file mode 100644
index 000000000000..8df808beae69
--- /dev/null
+++ b/maintainers/scripts/update-ruby-packages.checks.nix
@@ -0,0 +1,210 @@
+{
+  old,
+  new,
+  specifiedGems,
+  withData ? false,
+  # Use for `lib`.
+  pkgs ? import ../.. { },
+}:
+
+# Rename inputs to re-use those names.
+let
+  old' = old;
+  new' = new;
+  specifiedGems' = specifiedGems;
+in
+
+let
+  inherit (builtins)
+    attrNames
+    concatStrings
+    filter
+    genList
+    isNull
+    length
+    stringLength
+    toJSON
+    ;
+  inherit (pkgs.lib)
+    concatMapStringsSep
+    concatStringsSep
+    intersectLists
+    splitString
+    subtractLists
+    versionOlder
+    ;
+
+  # Keeps non-nulls in a list.
+  # Mirroring Ruby's `Array#compact`.
+  compact = filter (v: !(isNull v));
+
+  # The full gemsets attribute sets.
+  old = import old';
+  new = import new';
+
+  # All gem names.
+  allGems = attrNames (old // new);
+
+  # Gems found in both old and new.
+  keptGems = intersectLists (attrNames old) (attrNames new);
+
+  # Gems added or removed.
+  addedOrRemovedGems = subtractLists keptGems allGems;
+
+  # Gems specified in Gemfile.
+  specifiedGems = splitString "\n" specifiedGems';
+
+  # Gems that were not specified.
+  nonSpecifiedGems = subtractLists specifiedGems keptGems;
+
+  # Generates data for the summary tables
+  # This is also used for `failedChecks`.
+  versionChangeDataFor =
+    gems:
+    let
+      results = map (
+        name:
+        let
+          oldv = old.${name}.version or null;
+          newv = new.${name}.version or null;
+        in
+        if newv == oldv then
+          # Nothing changed. This will be filtered out.
+          null
+        else
+          {
+            inherit
+              name
+              ;
+            old = oldv;
+            new = newv;
+          }
+      ) gems;
+    in
+    compact results;
+
+  checkRegression =
+    entry: message:
+    let
+      isRemoval = isNull entry.new;
+      isAddition = isNull entry.old;
+      isRegression = versionOlder entry.new entry.old;
+    in
+    if
+      # Gems being added or gems being removed won't cause failures.
+      !isRemoval
+      && !isAddition
+      # A version being regressed is a failure.
+      && isRegression
+    then
+      message
+    else
+      null;
+
+  # This is a list of error messages to float up to the user.
+  # An empty list means no error.
+  failedChecks = compact (
+    [ ]
+    ++ (map (
+      entry:
+      checkRegression entry "Version regression for specified gem ${toJSON entry.name}, from ${toJSON entry.old} to ${toJSON entry.new}"
+    ) (versionChangeDataFor specifiedGems))
+    ++ (map (
+      entry:
+      checkRegression entry "Version regression for non-specified gem ${toJSON entry.name}, from ${toJSON entry.old} to ${toJSON entry.new}"
+    ) (versionChangeDataFor nonSpecifiedGems))
+  );
+
+  # Formats a version number (or null) as markdown.
+  gemVersionToMD = version: if isNull version then "*N/A*" else "`${version}`";
+
+  # Formats a `versionChangeDataFor` output as markdown.
+  versionChangeDataMD =
+    gems:
+    let
+      result = versionChangeDataFor gems;
+    in
+    map (
+      row:
+      [ row.name ]
+      ++ (map gemVersionToMD [
+        row.old
+        row.new
+      ])
+    ) result;
+
+  # Given a list of columns, and a list of list of column data,
+  # generates the markup for markdown table.
+  mkTable =
+    columns: entries:
+    let
+      entryToMarkdown = columns: "| ${concatStringsSep " | " columns} |";
+      sep = entryToMarkdown (map (_: "---") columns);
+    in
+    if length entries == 0 then
+      "> *No data...*"
+    else
+      ''
+        ${entryToMarkdown columns}
+        ${sep}
+        ${concatMapStringsSep "\n" entryToMarkdown entries}
+      '';
+
+  # The markdown report is built as this string.
+  report = ''
+    <!--
+    ----------------------------------------------
+    NOTE: You must copy this whole report section
+          to your pull request!
+    ----------------------------------------------
+    -->
+
+    #### Nixpkgs Ruby packages update report
+
+    **Specified gems changed:**
+
+    ${mkTable [ "Name" "old" "new" ] (versionChangeDataMD specifiedGems)}
+
+    **Gems added or removed:**
+
+    ${mkTable [ "Name" "old" "new" ] (versionChangeDataMD addedOrRemovedGems)}
+
+    <details>
+
+    <summary><strong>(Non-specified gem changes)</strong></summary>
+
+    ${mkTable [ "Name" "old" "new" ] (versionChangeDataMD nonSpecifiedGems)}
+
+    </details>
+
+    <!-- --------------- End ----------------- -->
+  '';
+in
+if (length failedChecks) > 0 then
+  # Fail the update script via `abort` on checks failure.
+  builtins.abort ''
+    ${"\n"}Gem upgrade aborted with the following failures:
+
+    ${concatMapStringsSep "\n" (msg: " - ${msg}") failedChecks}
+  ''
+else
+  # Output the report.
+  builtins.trace "(Report follows...)\n\n${report}" (
+    # And if `withData` is true, expose the data for REPL usage.
+    if withData then
+      {
+        inherit
+          # The gemsets used
+          old
+          new
+          # The lists of gems
+          allGems
+          specifiedGems
+          nonSpecifiedGems
+          addedOrRemovedGems
+          keptGems
+          ;
+      }
+    else
+      null
+  )