Merge branch 'master' into staging-next

34 files changed, 351 insertions, 250 deletions

diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix
index ec21dd9fafb3..5b7eec82c7a1 100644
--- a/maintainers/maintainer-list.nix
+++ b/maintainers/maintainer-list.nix
@@ -2884,6 +2884,15 @@
     githubId = 137306;
     name = "Michele Catalano";
   };
+  isgy = {
+    email = "isgy@teiyg.com";
+    github = "isgy";
+    githubId = 13622947;
+    keys = [{
+      longkeyid = "rsa4096/0xD3E1B013B4631293";
+      fingerprint = "1412 816B A9FA F62F D051 1975 D3E1 B013 B463 1293";
+    }];
+  };
   ivan = {
     email = "ivan@ludios.org";
     github = "ivan";
diff --git a/nixos/lib/test-driver/test-driver.py b/nixos/lib/test-driver/test-driver.py
index c8940d78af4e..c8d4936ac52a 100644
--- a/nixos/lib/test-driver/test-driver.py
+++ b/nixos/lib/test-driver/test-driver.py
@@ -381,15 +381,17 @@ class Machine:
 
     def succeed(self, *commands):
         """Execute each command and check that it succeeds."""
+        output = ""
         for command in commands:
             with self.nested("must succeed: {}".format(command)):
-                status, output = self.execute(command)
+                (status, out) = self.execute(command)
                 if status != 0:
-                    self.log("output: {}".format(output))
+                    self.log("output: {}".format(out))
                     raise Exception(
                         "command `{}` failed (exit code {})".format(command, status)
                     )
-                return output
+                output += out
+        return output
 
     def fail(self, *commands):
         """Execute each command and check that it fails."""
diff --git a/nixos/modules/installer/cd-dvd/installation-cd-graphical-base.nix b/nixos/modules/installer/cd-dvd/installation-cd-graphical-base.nix
index 719ba5ffb127..e0b558dcb0d8 100644
--- a/nixos/modules/installer/cd-dvd/installation-cd-graphical-base.nix
+++ b/nixos/modules/installer/cd-dvd/installation-cd-graphical-base.nix
@@ -22,17 +22,7 @@ with lib;
     });
   '';
 
-  services.xserver = {
-    enable = true;
-
-    # Automatically login as nixos.
-    displayManager.slim = {
-      enable = true;
-      defaultUser = "nixos";
-      autoLogin = true;
-    };
-
-  };
+  services.xserver.enable = true;
 
   # Provide networkmanager for easy wireless configuration.
   networking.networkmanager.enable = true;
diff --git a/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix b/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix
index 0b813bbf37b4..23c3426bff08 100644
--- a/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix
+++ b/nixos/modules/installer/cd-dvd/installation-cd-graphical-gnome.nix
@@ -10,8 +10,6 @@ with lib;
 
   services.xserver.desktopManager.gnome3.enable = true;
 
-  services.xserver.displayManager.slim.enable = mkForce false;
-
   # Auto-login as root.
   services.xserver.displayManager.gdm.autoLogin = {
     enable = true;
diff --git a/nixos/modules/installer/cd-dvd/installation-cd-graphical-kde.nix b/nixos/modules/installer/cd-dvd/installation-cd-graphical-kde.nix
index 1dc7920ff640..6a10a6404e67 100644
--- a/nixos/modules/installer/cd-dvd/installation-cd-graphical-kde.nix
+++ b/nixos/modules/installer/cd-dvd/installation-cd-graphical-kde.nix
@@ -13,6 +13,15 @@ with lib;
       enable = true;
       enableQt4Support = false;
     };
+
+    # Automatically login as nixos.
+    displayManager.sddm = {
+      enable = true;
+      autoLogin = {
+        enable = true;
+        user = "nixos";
+      };
+    };
   };
 
   environment.systemPackages = with pkgs; [
diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix
index 656a2d21b868..543a7b25d5d6 100644
--- a/nixos/modules/services/network-filesystems/ceph.nix
+++ b/nixos/modules/services/network-filesystems/ceph.nix
@@ -9,12 +9,14 @@ let
   expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
   expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
 
-  makeServices = (daemonType: daemonIds: extraServiceConfig:
+  makeServices = (daemonType: daemonIds:
     mkMerge (map (daemonId:
-      { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; })
+      { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
       daemonIds));
 
-  makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
+  makeService = (daemonType: daemonId: clusterName: ceph:
+    let
+      stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in {
     enable = true;
     description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
     after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
@@ -22,6 +24,11 @@ let
     partOf = [ "ceph-${daemonType}.target" ];
     wantedBy = [ "ceph-${daemonType}.target" ];
 
+    path = [ pkgs.getopt ];
+
+    # Don't start services that are not yet initialized
+    unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
+
     serviceConfig = {
       LimitNOFILE = 1048576;
       LimitNPROC = 1048576;
@@ -34,22 +41,22 @@ let
       Restart = "on-failure";
       StartLimitBurst = "5";
       StartLimitInterval = "30min";
+      StateDirectory = stateDirectory;
+      User = "ceph";
+      Group = if daemonType == "osd" then "disk" else "ceph";
       ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
-                    -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \
-                    --setgroup ${if daemonType == "osd" then "disk" else "ceph"}'';
-    } // extraServiceConfig
-      // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \
-                                                              --id ${daemonId} --cluster ${clusterName}''; };
-    } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) {
-      preStart = ''
-        daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"
-        if [ ! -d $daemonPath ]; then
-          mkdir -m 755 -p $daemonPath
-          chown -R ceph:ceph $daemonPath
-        fi
-      '';
-    } // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; }
-  );
+                    -f --cluster ${clusterName} --id ${daemonId}'';
+    } // optionalAttrs (daemonType == "osd") {
+      ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}'';
+      StartLimitBurst = "30";
+      RestartSec = "20s";
+      PrivateDevices = "no"; # osd needs disk access
+    } // optionalAttrs ( daemonType == "mon") {
+      RestartSec = "10";
+    } // optionalAttrs (lib.elem daemonType ["mgr" "mds"]) {
+      StartLimitBurst = "3";
+    };
+  });
 
   makeTarget = (daemonType:
     {
@@ -58,6 +65,7 @@ let
         partOf = [ "ceph.target" ];
         wantedBy = [ "ceph.target" ];
         before = [ "ceph.target" ];
+        unitConfig.StopWhenUnneeded = true;
       };
     }
   );
@@ -377,22 +385,22 @@ in
 
     systemd.services = let
       services = []
-        ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; })
-        ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
-        ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30";
-                                                                         RestartSec = "20s";
-                                                                         PrivateDevices = "no"; # osd needs disk access
-                                                                       })
-        ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { })
-        ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
+        ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
+        ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
+        ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
+        ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
+        ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
       in
         mkMerge services;
 
     systemd.targets = let
       targets = [
-        { ceph = { description = "Ceph target allowing to start/stop all ceph service instances at once";
-                     wantedBy = [ "multi-user.target" ]; }; }
-      ] ++ optional cfg.mon.enable (makeTarget "mon")
+        { ceph = {
+          description = "Ceph target allowing to start/stop all ceph service instances at once";
+          wantedBy = [ "multi-user.target" ];
+          unitConfig.StopWhenUnneeded = true;
+        }; } ]
+        ++ optional cfg.mon.enable (makeTarget "mon")
         ++ optional cfg.mds.enable (makeTarget "mds")
         ++ optional cfg.osd.enable (makeTarget "osd")
         ++ optional cfg.rgw.enable (makeTarget "rgw")
@@ -401,7 +409,11 @@ in
         mkMerge targets;
 
     systemd.tmpfiles.rules = [
+      "d /etc/ceph - ceph ceph - -"
       "d /run/ceph 0770 ceph ceph -"
-    ];
+      "d /var/lib/ceph - ceph ceph - -"]
+    ++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"]
+    ++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"]
+    ++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"];
   };
 }
diff --git a/nixos/modules/services/web-servers/phpfpm/default.nix b/nixos/modules/services/web-servers/phpfpm/default.nix
index 4ab7e3f0c0a9..095de64dfb14 100644
--- a/nixos/modules/services/web-servers/phpfpm/default.nix
+++ b/nixos/modules/services/web-servers/phpfpm/default.nix
@@ -31,7 +31,7 @@ let
     '';
     passAsFile = [ "nixDefaults" "phpOptions" ];
   } ''
-    cat $phpPackage/etc/php.ini $nixDefaultsPath $phpOptionsPath > $out
+    cat ${poolOpts.phpPackage}/etc/php.ini $nixDefaultsPath $phpOptionsPath > $out
   '';
 
   poolOpts = { name, ... }:
@@ -69,8 +69,6 @@ let
 
         phpOptions = mkOption {
           type = types.lines;
-          default = cfg.phpOptions;
-          defaultText = "config.services.phpfpm.phpOptions";
           description = ''
             "Options appended to the PHP configuration file <filename>php.ini</filename> used for this PHP-FPM pool."
           '';
@@ -137,6 +135,7 @@ let
       config = {
         socket = if poolOpts.listen == "" then "${runtimeDir}/${name}.sock" else poolOpts.listen;
         group = mkDefault poolOpts.user;
+        phpOptions = mkBefore cfg.phpOptions;
 
         settings = mapAttrs (name: mkDefault){
           listen = poolOpts.socket;
diff --git a/nixos/tests/ceph-multi-node.nix b/nixos/tests/ceph-multi-node.nix
index 6698aac3f271..ed493d6a1b34 100644
--- a/nixos/tests/ceph-multi-node.nix
+++ b/nixos/tests/ceph-multi-node.nix
@@ -49,9 +49,6 @@ let
     boot.kernelModules = [ "xfs" ];
 
     services.ceph = cephConfig;
-
-    # So that we don't have to battle systemd when bootstraping
-    systemd.targets.ceph.wantedBy = lib.mkForce [];
   };
 
   networkMonA = {
@@ -107,6 +104,10 @@ let
     };
   }; };
 
+  # Following deployment is based on the manual deployment described here:
+  # https://docs.ceph.com/docs/master/install/manual-deployment/
+  # For other ways to deploy a ceph cluster, look at the documentation at
+  # https://docs.ceph.com/docs/master/
   testscript = { ... }: ''
     startAll;
 
@@ -114,27 +115,6 @@ let
     $osd0->waitForUnit("network.target");
     $osd1->waitForUnit("network.target");
 
-    # Create the ceph-related directories
-    $monA->mustSucceed(
-      "mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}",
-      "mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}",
-      "chown ceph:ceph -R /var/lib/ceph/",
-      "mkdir -p /etc/ceph",
-      "chown ceph:ceph -R /etc/ceph"
-    );
-    $osd0->mustSucceed(
-      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
-      "chown ceph:ceph -R /var/lib/ceph/",
-      "mkdir -p /etc/ceph",
-      "chown ceph:ceph -R /etc/ceph"
-    );
-    $osd1->mustSucceed(
-      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
-      "chown ceph:ceph -R /var/lib/ceph/",
-      "mkdir -p /etc/ceph",
-      "chown ceph:ceph -R /etc/ceph"
-    );
-
     # Bootstrap ceph-mon daemon
     $monA->mustSucceed(
       "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
@@ -142,6 +122,7 @@ let
       "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
       "monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
       "sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
+      "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
       "sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
       "systemctl start ceph-mon-${cfg.monA.name}"
     );
@@ -168,12 +149,14 @@ let
     # Bootstrap both OSDs
     $osd0->mustSucceed(
       "mkfs.xfs /dev/vdb",
+      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
       "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
       "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
       "echo '{\"cephx_secret\": \"${cfg.osd0.key}\"}' | ceph osd new ${cfg.osd0.uuid} -i -",
     );
     $osd1->mustSucceed(
       "mkfs.xfs /dev/vdb",
+      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
       "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
       "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
       "echo '{\"cephx_secret\": \"${cfg.osd1.key}\"}' | ceph osd new ${cfg.osd1.uuid} -i -"
@@ -209,22 +192,17 @@ let
       "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it"
     );
 
-    # As we disable the target in the config, we still want to test that it works as intended
-    $osd0->mustSucceed("systemctl stop ceph-osd-${cfg.osd0.name}");
-    $osd1->mustSucceed("systemctl stop ceph-osd-${cfg.osd1.name}");
-    $monA->mustSucceed(
-      "systemctl stop ceph-mgr-${cfg.monA.name}",
-      "systemctl stop ceph-mon-${cfg.monA.name}"
-    );
-    
-    $monA->succeed("systemctl start ceph.target");
-    $monA->waitForUnit("ceph-mon-${cfg.monA.name}");
-    $monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
-    $osd0->succeed("systemctl start ceph.target");
-    $osd0->waitForUnit("ceph-osd-${cfg.osd0.name}");
-    $osd1->succeed("systemctl start ceph.target");
-    $osd1->waitForUnit("ceph-osd-${cfg.osd1.name}");
-    
+    # Shut down ceph on all machines in a very unpolite way
+    $monA->crash;
+    $osd0->crash;
+    $osd1->crash;
+
+    # Start it up
+    $osd0->start;
+    $osd1->start;
+    $monA->start;
+
+    # Ensure the cluster comes back up again
     $monA->succeed("ceph -s | grep 'mon: 1 daemons'");
     $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
     $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
diff --git a/nixos/tests/ceph-single-node.nix b/nixos/tests/ceph-single-node.nix
index 10b77cff5a31..041fbd7e8e64 100644
--- a/nixos/tests/ceph-single-node.nix
+++ b/nixos/tests/ceph-single-node.nix
@@ -46,9 +46,6 @@ let
     boot.kernelModules = [ "xfs" ];
 
     services.ceph = cephConfig;
-
-    # So that we don't have to battle systemd when bootstraping
-    systemd.targets.ceph.wantedBy = lib.mkForce [];
   };
 
   networkMonA = {
@@ -72,22 +69,15 @@ let
     };
   }; };
 
+  # Following deployment is based on the manual deployment described here:
+  # https://docs.ceph.com/docs/master/install/manual-deployment/
+  # For other ways to deploy a ceph cluster, look at the documentation at
+  # https://docs.ceph.com/docs/master/
   testscript = { ... }: ''
     startAll;
 
     $monA->waitForUnit("network.target");
 
-    # Create the ceph-related directories
-    $monA->mustSucceed(
-      "mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}",
-      "mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}",
-      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
-      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
-      "mkdir -p /etc/ceph",
-      "chown ceph:ceph -R /etc/ceph",
-      "chown ceph:ceph -R /var/lib/ceph/",
-    );
-
     # Bootstrap ceph-mon daemon
     $monA->mustSucceed(
       "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
@@ -104,8 +94,9 @@ let
     # Can't check ceph status until a mon is up
     $monA->succeed("ceph -s | grep 'mon: 1 daemons'");
 
-    # Start the ceph-mgr daemon, it has no deps and hardly any setup
+    # Start the ceph-mgr daemon, after copying in the keyring
     $monA->mustSucceed(
+      "sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
       "ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
       "systemctl start ceph-mgr-${cfg.monA.name}"
     );
@@ -117,7 +108,9 @@ let
     $monA->mustSucceed(
       "mkfs.xfs /dev/vdb",
       "mkfs.xfs /dev/vdc",
+      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
       "mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
+      "mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
       "mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
       "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
       "ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
@@ -159,20 +152,17 @@ let
       "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it"
     );
 
-    # As we disable the target in the config, we still want to test that it works as intended
-    $monA->mustSucceed(
-      "systemctl stop ceph-osd-${cfg.osd0.name}",
-      "systemctl stop ceph-osd-${cfg.osd1.name}",
-      "systemctl stop ceph-mgr-${cfg.monA.name}",
-      "systemctl stop ceph-mon-${cfg.monA.name}"
-    );
-    
+    # Shut down ceph by stopping ceph.target.
+    $monA->mustSucceed("systemctl stop ceph.target");
+
+    # Start it up
     $monA->succeed("systemctl start ceph.target");
     $monA->waitForUnit("ceph-mon-${cfg.monA.name}");
     $monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
     $monA->waitForUnit("ceph-osd-${cfg.osd0.name}");
     $monA->waitForUnit("ceph-osd-${cfg.osd1.name}");
-    
+
+    # Ensure the cluster comes back up again
     $monA->succeed("ceph -s | grep 'mon: 1 daemons'");
     $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
     $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
diff --git a/nixos/tests/xmonad.nix b/nixos/tests/xmonad.nix
index 79c15ccffecd..c2e5ba60d7bc 100644
--- a/nixos/tests/xmonad.nix
+++ b/nixos/tests/xmonad.nix
@@ -1,4 +1,4 @@
-import ./make-test.nix ({ pkgs, ...} : {
+import ./make-test-python.nix ({ pkgs, ...} : {
   name = "xmonad";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ nequissimus ];
@@ -21,19 +21,21 @@ import ./make-test.nix ({ pkgs, ...} : {
     };
   };
 
-  testScript = { ... }: ''
-    $machine->waitForX;
-    $machine->waitForFile("/home/alice/.Xauthority");
-    $machine->succeed("xauth merge ~alice/.Xauthority");
-    $machine->sendKeys("alt-ctrl-x");
-    $machine->waitForWindow(qr/alice.*machine/);
-    $machine->sleep(1);
-    $machine->screenshot("terminal");
-    $machine->waitUntilSucceeds("xmonad --restart");
-    $machine->sleep(3);
-    $machine->sendKeys("alt-shift-ret");
-    $machine->waitForWindow(qr/alice.*machine/);
-    $machine->sleep(1);
-    $machine->screenshot("terminal");
+  testScript = { nodes, ... }: let
+    user = nodes.machine.config.users.users.alice;
+  in ''
+    machine.wait_for_x()
+    machine.wait_for_file("${user.home}/.Xauthority")
+    machine.succeed("xauth merge ${user.home}/.Xauthority")
+    machine.send_chars("alt-ctrl-x")
+    machine.wait_for_window("${user.name}.*machine")
+    machine.sleep(1)
+    machine.screenshot("terminal")
+    machine.wait_until_succeeds("xmonad --restart")
+    machine.sleep(3)
+    machine.send_chars("alt-shift-ret")
+    machine.wait_for_window("${user.name}.*machine")
+    machine.sleep(1)
+    machine.screenshot("terminal")
   '';
 })
diff --git a/pkgs/applications/graphics/drawio/default.nix b/pkgs/applications/graphics/drawio/default.nix
index 4142310b705a..25cefabbd634 100644
--- a/pkgs/applications/graphics/drawio/default.nix
+++ b/pkgs/applications/graphics/drawio/default.nix
@@ -11,11 +11,11 @@
 
 stdenv.mkDerivation rec {
   pname = "drawio";
-  version = "12.1.7";
+  version = "12.2.2";
 
   src = fetchurl {
     url = "https://github.com/jgraph/drawio-desktop/releases/download/v${version}/draw.io-x86_64-${version}.rpm";
-    sha256 = "1vac0cz99yjlz7b186wyy4wk4sxkvlirpjmh3vw65xaxamn8spn3";
+    sha256 = "04h11gdy78py9zrs3v6y0hhhc2n1h4s0ymbvf6qn4vv4r3r9vbaw";
   };
 
   nativeBuildInputs = [
diff --git a/pkgs/applications/misc/wtf/default.nix b/pkgs/applications/misc/wtf/default.nix
index 2706a5ed658a..f20c4aff1697 100644
--- a/pkgs/applications/misc/wtf/default.nix
+++ b/pkgs/applications/misc/wtf/default.nix
@@ -7,23 +7,21 @@
 
 buildGoModule rec {
   pname = "wtf";
-  version = "0.23.0";
-
-  overrideModAttrs = _oldAttrs : _oldAttrs // {
-    preBuild = ''export GOPROXY="https://gocenter.io"'';
-  };
+  version = "0.24.0";
 
   src = fetchFromGitHub {
     owner = "wtfutil";
     repo = pname;
     rev = "v${version}";
-    sha256 = "0bhk81jmv6rq8h898lmvrh9v356310fbi82lvakmgay7nvzk9a1c";
+    sha256 = "0jz7hjcm0hfxcih2zplp47wx6lyvhhzj9ka4ljqrx0i4l7cm9ahs";
    };
 
-  modSha256 = "1ndb7zbhaq0cnd8fd05fvb62qi0mxilgydz42qqz2z4fkbx9gp3m";
+  modSha256 = "04d8hvd90f7v853p23xcx38qz3ryv7kz7zjk9b131cjnd4mcv0sm";
 
   buildFlagsArray = [ "-ldflags=-s -w -X main.version=${version}" ];
 
+  subPackages = [ "." ];
+
   nativeBuildInputs = [ makeWrapper ];
 
   postInstall = ''
diff --git a/pkgs/applications/networking/calls/default.nix b/pkgs/applications/networking/calls/default.nix
index 63caf836ad9e..065f264f1529 100644
--- a/pkgs/applications/networking/calls/default.nix
+++ b/pkgs/applications/networking/calls/default.nix
@@ -9,24 +9,26 @@
 , gom
 , gsound
 , evolution-data-server
+, folks
 , desktop-file-utils
 , libpeas
 , dbus
+, vala
 , xorg
 , xvfb_run
 , libxml2
 }:
 
 stdenv.mkDerivation rec {
-  pname = "calls-unstable";
-  version = "2019-10-09";
+  pname = "calls";
+  version = "unstable-2019-10-29";
 
   src = fetchFromGitLab {
     domain = "source.puri.sm";
     owner = "Librem5";
-    repo = pname;
-    rev = "4b4cfa04266ebbe2f3da5abd9624ea07aa159fea";
-    sha256 = "0qvnddjpkh6gsywzdi24lmjlbwi0q54m1xa6hiaf1ch1j7kcv8fr";
+    repo = "calls";
+    rev = "9fe575053d8f01c3a76a6c20d39f0816166d5afd";
+    sha256 = "01inx4mvrzvklwrfryw5hw9p89v8cn78m3qmv97g7a3v0h5c0n35";
   };
 
   nativeBuildInputs = [
@@ -34,27 +36,29 @@ stdenv.mkDerivation rec {
     ninja
     pkgconfig
     desktop-file-utils
+    vala
   ];
 
   buildInputs = [
     modemmanager
     libhandy
     evolution-data-server
+    folks
     gom
     gsound
     gtk3
     libhandy
     libpeas
-    libxml2
   ];
 
   checkInputs = [
     dbus
     xvfb_run
-    xorg.xauth
   ];
 
   mesonFlags = [
+    # docs fail to build
+    # https://source.puri.sm/Librem5/calls/issues/99
     "-Dgtk_doc=false"
   ];
 
diff --git a/pkgs/development/haskell-modules/patches/ghc-paths-nix-ghcjs.patch b/pkgs/development/haskell-modules/patches/ghc-paths-nix-ghcjs.patch
index 26ea0eaa5a8b..48cf6a035afb 100644
--- a/pkgs/development/haskell-modules/patches/ghc-paths-nix-ghcjs.patch
+++ b/pkgs/development/haskell-modules/patches/ghc-paths-nix-ghcjs.patch
@@ -43,13 +43,13 @@ index c87565d..88b3db4 100644
 +ghc     = fromMaybe GHC_PATHS_GHC     nixGhc
 +ghc_pkg = fromMaybe GHC_PATHS_GHC_PKG nixGhcPkg
 diff --git a/Setup.hs b/Setup.hs
-index fad5026..1651650 100644
+index f2d1733..ca4792e 100644
 --- a/Setup.hs
 +++ b/Setup.hs
-@@ -27,13 +27,13 @@ main = defaultMainWithHooks simpleUserHooks {
-     defaultPostConf :: Args -> ConfigFlags -> PackageDescription -> LocalBuildInfo -> IO ()
-     defaultPostConf args flags pkgdescr lbi = do
+@@ -39,13 +39,13 @@ main = defaultMainWithHooks simpleUserHooks {
+ #else
        libdir_ <- rawSystemProgramStdoutConf (fromFlag (configVerbosity flags))
+ #endif
 -                     ghcProgram (withPrograms lbi) ["--print-libdir"]
 +                     ghcjsProgram (withPrograms lbi) ["--print-libdir"]
        let libdir = reverse $ dropWhile isSpace $ reverse libdir_
diff --git a/pkgs/development/libraries/rocksdb/default.nix b/pkgs/development/libraries/rocksdb/default.nix
index d2ea22b85a1a..2bb56bec8c0a 100644
--- a/pkgs/development/libraries/rocksdb/default.nix
+++ b/pkgs/development/libraries/rocksdb/default.nix
@@ -30,6 +30,7 @@ stdenv.mkDerivation rec {
     "-DWITH_ZLIB=1"
     "-DWITH_ZSTD=1"
     "-DWITH_GFLAGS=0"
+    "-DUSE_RTTI=1"
     (lib.optional
         (stdenv.hostPlatform.system == "i686-linux"
          || stdenv.hostPlatform.system == "x86_64-linux")
diff --git a/pkgs/development/python-modules/datasette/default.nix b/pkgs/development/python-modules/datasette/default.nix
index 828976f0e9fb..7210aa6b1e31 100644
--- a/pkgs/development/python-modules/datasette/default.nix
+++ b/pkgs/development/python-modules/datasette/default.nix
@@ -20,16 +20,16 @@
 
 buildPythonPackage rec {
   pname = "datasette";
-  version = "0.29.3";
+  version = "0.30.2";
 
   src = fetchFromGitHub {
     owner = "simonw";
     repo = "datasette";
     rev = version;
-    sha256 = "0cib7pd4z240ncck0pskzvizblhwkr42fsjpd719wdxy4scs7yqa";
+    sha256 = "07swnpz4c7vzlc69vavs1xvbhr5fa8g63kyfj1hf3zafskgjnzwy";
   };
 
-  buildInputs = [ pytestrunner ];
+  nativeBuildInputs = [ pytestrunner ];
 
   propagatedBuildInputs = [
     click
@@ -57,15 +57,19 @@ buildPythonPackage rec {
       --replace "Sanic==0.7.0" "Sanic" \
       --replace "hupper==1.0" "hupper" \
       --replace "pint~=0.8.1" "pint" \
+      --replace "pluggy~=0.12.0" "pint" \
       --replace "Jinja2==2.10.1" "Jinja2" \
       --replace "uvicorn~=0.8.4" "uvicorn"
   '';
 
   # many tests require network access
+  # test_black fails on darwin
   checkPhase = ''
     pytest --ignore tests/test_api.py \
            --ignore tests/test_csv.py \
-           --ignore tests/test_html.py
+           --ignore tests/test_html.py \
+           --ignore tests/test_black.py \
+           -k 'not facet'
   '';
 
   meta = with lib; {
diff --git a/pkgs/development/python-modules/google_cloud_vision/default.nix b/pkgs/development/python-modules/google_cloud_vision/default.nix
index a8d748db126d..10c8792b9c86 100644
--- a/pkgs/development/python-modules/google_cloud_vision/default.nix
+++ b/pkgs/development/python-modules/google_cloud_vision/default.nix
@@ -3,7 +3,6 @@
 , fetchPypi
 , enum34
 , google_api_core
-, pytest
 , mock
 }:
 
@@ -16,11 +15,13 @@ buildPythonPackage rec {
     sha256 = "f33aea6721d453901ded268dee61a01ab77d4cd215a76edc3cc61b6028299d3e";
   };
 
-  checkInputs = [ pytest mock ];
+  checkInputs = [ mock ];
   propagatedBuildInputs = [ enum34 google_api_core ];
 
+  # pytest seems to pick up some file which overrides PYTHONPATH
   checkPhase = ''
-    pytest tests/unit
+    cd tests/unit
+    python -m unittest discover
   '';
 
   meta = with stdenv.lib; {
diff --git a/pkgs/development/python-modules/jsonlines/default.nix b/pkgs/development/python-modules/jsonlines/default.nix
new file mode 100644
index 000000000000..eec4c6b38469
--- /dev/null
+++ b/pkgs/development/python-modules/jsonlines/default.nix
@@ -0,0 +1,28 @@
+{ lib, fetchFromGitHub, buildPythonPackage, six
+, flake8, pep8-naming, pytest, pytestcov, pytestpep8 }:
+
+buildPythonPackage rec {
+  pname = "jsonlines";
+  version = "1.2.0";
+
+  src = fetchFromGitHub {
+    owner = "wbolster";
+    repo = pname;
+    rev = version;
+    sha256 = "1f8zsqy8p9a41gqg2a5x7sppc5qhhq7gw58id2aigb270yxzs7jw";
+  };
+
+  propagatedBuildInputs = [ six ];
+
+  checkInputs = [ flake8 pep8-naming pytest pytestcov pytestpep8 ];
+  checkPhase = ''
+    pytest
+  '';
+
+  meta = with lib; {
+    description = "Python library to simplify working with jsonlines and ndjson data";
+    homepage = https://github.com/wbolster/jsonlines;
+    maintainers = with maintainers; [ sondr3 ];
+    license = licenses.bsd3;
+  };
+}
diff --git a/pkgs/development/python-modules/jupyterlab_server/default.nix b/pkgs/development/python-modules/jupyterlab_server/default.nix
index 6d0da2925cc2..edfe01728a49 100644
--- a/pkgs/development/python-modules/jupyterlab_server/default.nix
+++ b/pkgs/development/python-modules/jupyterlab_server/default.nix
@@ -6,6 +6,7 @@
 , pythonOlder
 , requests
 , pytest
+, pyjson5
 }:
 
 buildPythonPackage rec {
@@ -19,7 +20,7 @@ buildPythonPackage rec {
   };
 
   checkInputs = [ requests pytest ];
-  propagatedBuildInputs = [ notebook jsonschema ];
+  propagatedBuildInputs = [ notebook jsonschema pyjson5 ];
 
   # test_listing test fails
   # this is a new package and not all tests pass
diff --git a/pkgs/development/python-modules/pyjson5/default.nix b/pkgs/development/python-modules/pyjson5/default.nix
new file mode 100644
index 000000000000..232bafd3ba7e
--- /dev/null
+++ b/pkgs/development/python-modules/pyjson5/default.nix
@@ -0,0 +1,26 @@
+{ buildPythonPackage, lib, nose, fetchFromGitHub }:
+
+buildPythonPackage rec {
+  pname = "pyjson5";
+  version = "0.8.5";
+
+  src = fetchFromGitHub {
+    owner = "dpranke";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "0nyngj18jlkgvm1177lc3cj47wm4yh3dqigygvcvw7xkyryafsqn";
+  };
+
+  doCheck = true;
+  checkInputs = [ nose ];
+  checkPhase = ''
+    nosetests
+  '';
+
+  meta = with lib; {
+    description = "Python implementation of the JSON5 data format";
+    license = licenses.asl20;
+    homepage = "https://github.com/dpranke/pyjson5";
+    maintainers = with maintainers; [ isgy ];
+  };
+}
diff --git a/pkgs/development/python-modules/wurlitzer/default.nix b/pkgs/development/python-modules/wurlitzer/default.nix
index 8a4d96515d7c..b946d23acf60 100644
--- a/pkgs/development/python-modules/wurlitzer/default.nix
+++ b/pkgs/development/python-modules/wurlitzer/default.nix
@@ -1,8 +1,10 @@
 { lib
 , buildPythonPackage
 , fetchPypi
+, isPy27
 , mock
 , pytest
+, selectors2
 }:
 
 buildPythonPackage rec {
@@ -14,6 +16,8 @@ buildPythonPackage rec {
     sha256 = "0xndv47iwc9k8cp5r9r1z3r0xww0r5x5b7qsmn39gk2gsg0119c6";
   };
 
+  propagatedBuildInputs = lib.optionals isPy27 [ selectors2 ];
+
   checkInputs = [ mock pytest ];
 
   checkPhase = ''
@@ -25,4 +29,4 @@ buildPythonPackage rec {
     homepage = https://github.com/minrk/wurlitzer;
     license = lib.licenses.mit;
   };
-}
\ No newline at end of file
+}
diff --git a/pkgs/development/tools/analysis/hotspot/default.nix b/pkgs/development/tools/analysis/hotspot/default.nix
index 89ffbff3db71..fc7cd2ffe193 100644
--- a/pkgs/development/tools/analysis/hotspot/default.nix
+++ b/pkgs/development/tools/analysis/hotspot/default.nix
@@ -1,4 +1,5 @@
 { stdenv,
+  mkDerivation,
   cmake,
   elfutils,
   extra-cmake-modules,
@@ -14,7 +15,7 @@
   threadweaver,
 }:
 
-stdenv.mkDerivation rec {
+mkDerivation rec {
   pname = "hotspot";
   version = "1.2.0";
 
diff --git a/pkgs/games/steam/chrootenv.nix b/pkgs/games/steam/chrootenv.nix
index b2bca7ca7b97..0de961e6d49f 100644
--- a/pkgs/games/steam/chrootenv.nix
+++ b/pkgs/games/steam/chrootenv.nix
@@ -75,6 +75,7 @@ in buildFHSUserEnv rec {
     xorg.libX11
     xorg.libXfixes
     libGL
+    libva
 
     # Not formally in runtime but needed by some games
     at-spi2-atk
diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix
index 3a480bd74674..a40a142368ab 100644
--- a/pkgs/servers/search/elasticsearch/plugins.nix
+++ b/pkgs/servers/search/elasticsearch/plugins.nix
@@ -1,4 +1,4 @@
-{ pkgs, lib, stdenv, fetchurl, unzip, elasticsearch }:
+{ lib, stdenv, fetchurl, unzip, elasticsearch }:
 
 let
   esVersion = elasticsearch.version;
@@ -16,27 +16,27 @@ let
   }:
     stdenv.mkDerivation (a // {
       inherit installPhase;
+      pname = "elasticsearch-${pluginName}";
       dontUnpack = true;
       buildInputs = [ unzip ];
       meta = a.meta // {
         platforms = elasticsearch.meta.platforms;
-        maintainers = (a.meta.maintainers or []) ++ (with stdenv.lib.maintainers; [ offline ]);
+        maintainers = (a.meta.maintainers or []) ++ (with lib.maintainers; [ offline ]);
       };
     });
 in {
 
   analysis-lemmagen = esPlugin rec {
-    name = "elasticsearch-analysis-lemmagen-${version}";
-    pluginName = "elasticsearch-analysis-lemmagen";
+    pluginName = "analysis-lemmagen";
     version = esVersion;
     src = fetchurl {
-      url = "https://github.com/vhyza/${pluginName}/releases/download/v${version}/${name}-plugin.zip";
+      url = "https://github.com/vhyza/${pluginName}/releases/download/v${version}/${pluginName}-${version}-plugin.zip";
       sha256 =
-        if version == "7.0.1" then "155zj9zw81msx976c952nk926ndav1zqhmy2xih6nr82qf0p71hm"
-        else if version == "6.7.2" then "1r176ncjbilkmri2c5rdxh5xqsrn77m1f0p98zs47czwlqh230iq"
+        if version == "7.3.1" then "1nb82z6s94mzdx1srb1pwj7cpzs8w74njap0xiqn7sg5ylk6adm8"
+        else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = https://github.com/vhyza/elasticsearch-analysis-lemmagen;
       description = "LemmaGen Analysis plugin provides jLemmaGen lemmatizer as Elasticsearch token filter";
       license = licenses.asl20;
@@ -44,37 +44,50 @@ in {
   };
 
   discovery-ec2 = esPlugin rec {
-    name = "elasticsearch-discovery-ec2-${version}";
     pluginName = "discovery-ec2";
     version = esVersion;
-    src = pkgs.fetchurl {
+    src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.0.1" then "0nrvralh4fygs0ys2ikg3x08jdyh9276d5w7yfncbbi0xrg9hk6g"
-        else if version == "6.7.2" then "1p0cdz3lfksfd2kvlcj0syxhbx27mimsaw8q4kgjpjjjwqayg523"
-        else if version == "5.6.16" then "1300pfmnlpfm1hh2jgas8j2kqjqiqkxhr8czshj9lx0wl4ciknin"
+        if version == "7.3.1" then "1p30by7pqnvj8dcwws51kh9s962c42qwqq07gmj4jl83zxcl8kyl"
+        else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/discovery-ec2;
       description = "The EC2 discovery plugin uses the AWS API for unicast discovery.";
       license = licenses.asl20;
     };
   };
 
+  ingest-attachment = esPlugin rec {
+    pluginName = "ingest-attachment";
+    version = esVersion;
+    src = fetchurl {
+      url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
+      sha256 =
+        if version == "7.3.1" then "1b9l17zv6582sdcdiabwd293xx5ckc2d3h6smiv6znk5f4dxj7km"
+        else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
+        else throw "unsupported version ${version} for plugin ${pluginName}";
+    };
+    meta = with lib; {
+      homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/ingest-attachment;
+      description = "Ingest processor that uses Apache Tika to extract contents";
+      license = licenses.asl20;
+    };
+  };
+
   repository-s3 = esPlugin rec {
-    name = "elasticsearch-repository-s3-${version}";
     pluginName = "repository-s3";
     version = esVersion;
-    src = pkgs.fetchurl {
+    src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.0.1" then "17bf8m1q92j5yhgldckl4hlsfv6qgwwqdc1da9kzgidgky7jwkbc"
-        else if version == "6.7.2" then "1l353zfyv3qziz8xkann9cbzx4wj5s14wnknfw351j6vgdq26l12"
-        else if version == "5.6.16" then "0k3li5xv1270ygb9lqk6ji3nngngl2im3z38k08nd627vxdrzij2"
+        if version == "7.3.1" then "1dqd3hd8qa1bsvd1p42k5zcrdmb66d2yspfc7g8nsz89w6b1invg"
+        else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/repository-s3;
       description = "The S3 repository plugin adds support for using AWS S3 as a repository for Snapshot/Restore.";
       license = licenses.asl20;
@@ -82,18 +95,16 @@ in {
   };
 
   repository-gcs = esPlugin rec {
-    name = "elasticsearch-repository-gcs-${version}";
     pluginName = "repository-gcs";
     version = esVersion;
-    src = pkgs.fetchurl {
+    src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.0.1" then "0a3rc2gggsj7xfncil1s53dmq799lcm82h0yncs94jnb182sbmzc"
-        else if version == "6.7.2" then "0afccbvb7x6y3nrwmal09vpgxyz4lar6lffw4mngalcppsk8irvv"
-        else if version == "5.6.16" then "0hwqx4yhdn4c0ccdpvgrg30ag8hy3mgxgk7h7pibdmzvy7qw7501"
+        if version == "7.3.1" then "0kpb1hn2fb4lh6kn96vi7265ign9lwcd0zfc19l4n6fpp8js5lfh"
+        else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
-    meta = with stdenv.lib; {
+    meta = with lib; {
       homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/repository-gcs;
       description = "The GCS repository plugin adds support for using Google Cloud Storage as a repository for Snapshot/Restore.";
       license = licenses.asl20;
@@ -103,23 +114,21 @@ in {
   search-guard = let
     majorVersion = lib.head (builtins.splitVersion esVersion);
   in esPlugin rec {
-    name = "elasticsearch-search-guard-${version}";
     pluginName = "search-guard";
     version =
-      if esVersion == "7.0.1" then "${esVersion}-35.0.0"
-      else if esVersion == "6.7.2" then "${esVersion}-25.1"
-      else if esVersion == "5.6.16" then "${esVersion}-19.3"
+      # https://docs.search-guard.com/latest/search-guard-versions
+      if esVersion == "7.3.1" then "${esVersion}-37.0.0"
+      else if esVersion == "6.8.3" then "${esVersion}-25.5"
       else throw "unsupported version ${esVersion} for plugin ${pluginName}";
     src = fetchurl {
       url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
       sha256 =
-        if version == "7.0.1-35.0.0" then "0wsiqq7j7ph9g2vhhvjmwrh5a2q1wzlysgr75gc35zcvqz6cq8ha"
-        else if version == "6.7.2-25.1" then "119r1zibi0z40mfxrpkx0zzay0yz6c7syqmmw8i2681wmz4nksda"
-        else if version == "5.6.16-19.3" then "1q70anihh89c53fnk8wlq9z5dx094j0f9a0y0v2zsqx18lz9ikmx"
+        if version == "7.3.1-37.0.0" then "0rb631npr6vykrhln3x6q75xwb0wndvrspwnak0rld5d7pqn1r04"
+        else if version == "6.8.3-25.5" then "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
-    meta = with stdenv.lib; {
-      homepage = https://github.com/floragunncom/search-guard;
+    meta = with lib; {
+      homepage = https://search-guard.com;
       description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
       license = licenses.asl20;
     };
diff --git a/pkgs/shells/zsh/antibody/default.nix b/pkgs/shells/zsh/antibody/default.nix
index 0ea3c8d158d0..e1bfc68dc4de 100644
--- a/pkgs/shells/zsh/antibody/default.nix
+++ b/pkgs/shells/zsh/antibody/default.nix
@@ -2,7 +2,7 @@
 
 buildGoModule rec {
   pname = "antibody";
-  version = "4.1.2";
+  version = "4.2.0";
 
   goPackagePath = "github.com/getantibody/antibody";
 
@@ -10,15 +10,15 @@ buildGoModule rec {
     owner = "getantibody";
     repo = "antibody";
     rev = "v${version}";
-    sha256 = "1csanmvix7b2sa7nsy8nh3jq6gmhp8i51xivsabm1lj2y30c0ly3";
+    sha256 = "1vds7mxqxa7xlhvjvmnh1nr1ra3dciav0qlv45s1dmwn5qrcilci";
   };
 
-  modSha256 = "1p9cw92ivwgpkvjxvwd9anbd1vzhpicm9il4pg37z2kgr2ihhnyh";
+  modSha256 = "1n9sgrm16iig600f4q1cmbwwk0822isjvbyazplylha843510b17";
 
   meta = with lib; {
     description = "The fastest shell plugin manager";
     homepage = https://github.com/getantibody/antibody;
     license = licenses.mit;
-    maintainers = with maintainers; [ worldofpeace ];
+    maintainers = with maintainers; [ filalex77 worldofpeace ];
   };
 }
diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix
index 141d21951a88..3066be27772e 100644
--- a/pkgs/tools/filesystems/ceph/default.nix
+++ b/pkgs/tools/filesystems/ceph/default.nix
@@ -6,9 +6,10 @@
 , libxml2, zlib, lz4
 , openldap, lttng-ust
 , babeltrace, gperf
+, gtest
 , cunit, snappy
 , rocksdb, makeWrapper
-, leveldb, oathToolkit, removeReferencesTo
+, leveldb, oathToolkit
 
 # Optional Dependencies
 , yasm ? null, fcgi ? null, expat ? null
@@ -108,14 +109,14 @@ in rec {
     nativeBuildInputs = [
       cmake
       pkgconfig which git python3Packages.wrapPython makeWrapper
+      python3Packages.python # for the toPythonPath function
       (ensureNewerSourcesHook { year = "1980"; })
     ];
 
     buildInputs = cryptoLibsMap.${cryptoStr} ++ [
       boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3
-      malloc zlib openldap lttng-ust babeltrace gperf cunit
+      malloc zlib openldap lttng-ust babeltrace gperf gtest cunit
       snappy rocksdb lz4 oathToolkit leveldb
-      removeReferencesTo
     ] ++ optionals stdenv.isLinux [
       linuxHeaders utillinux libuuid udev keyutils optLibaio optLibxfs optZfs
       # ceph 14
@@ -124,54 +125,52 @@ in rec {
       optFcgi optExpat optCurl optFuse optLibedit
     ];
 
+    pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ];
+
     preConfigure =''
       substituteInPlace src/common/module.c --replace "/sbin/modinfo"  "modinfo"
       substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
-      # Since Boost 1.67 this seems to have changed
-      substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python37)"
-      substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python37 "
 
       # for pybind/rgw to find internal dep
       export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH"
       # install target needs to be in PYTHONPATH for "*.pth support" check to succeed
-      export PYTHONPATH=${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/python3.7/site-packages/:$out/lib/python3.7/site-packages/
 
-      patchShebangs src/spdk
+      patchShebangs src/script src/spdk src/test src/tools
     '';
 
     cmakeFlags = [
       "-DWITH_PYTHON3=ON"
       "-DWITH_SYSTEM_ROCKSDB=OFF"
+      "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
+
 
       "-DWITH_SYSTEM_BOOST=ON"
+      "-DWITH_SYSTEM_ROCKSDB=ON"
+      "-DWITH_SYSTEM_GTEST=ON"
+      "-DMGR_PYTHON_VERSION=${ceph-python-env.python.pythonVersion}"
       "-DWITH_SYSTEMD=OFF"
       "-DWITH_TESTS=OFF"
       # TODO breaks with sandbox, tries to download stuff with npm
       "-DWITH_MGR_DASHBOARD_FRONTEND=OFF"
     ];
 
-    preFixup = ''
-      find $lib -type f -exec remove-references-to -t $out '{}' +
-      mv $out/share/ceph/mgr $lib/lib/ceph/
-    '';
-
     postFixup = ''
-      export PYTHONPATH="${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
       wrapPythonPrograms
-      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
-      wrapProgram $out/bin/ceph-volume --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
+      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
     '';
 
     enableParallelBuilding = true;
 
     outputs = [ "out" "lib" "dev" "doc" "man" ];
 
+    doCheck = false; # uses pip to install things from the internet
+
     meta = {
       homepage = https://ceph.com/;
       description = "Distributed storage system";
       license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
       maintainers = with maintainers; [ adev ak krav johanot ];
-      platforms = platforms.unix;
+      platforms = [ "x86_64-linux" ];
     };
 
     passthru.version = version;
@@ -183,7 +182,7 @@ in rec {
         description = "Tools needed to mount Ceph's RADOS Block Devices";
         license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
         maintainers = with maintainers; [ adev ak johanot krav ];
-        platforms = platforms.unix;
+        platforms = [ "x86_64-linux" ];
       };
     } ''
       mkdir -p $out/{bin,etc,lib/python3.7/site-packages}
diff --git a/pkgs/tools/misc/licensor/default.nix b/pkgs/tools/misc/licensor/default.nix
new file mode 100644
index 000000000000..99d9fc72681e
--- /dev/null
+++ b/pkgs/tools/misc/licensor/default.nix
@@ -0,0 +1,23 @@
+{ lib, rustPlatform, fetchFromGitHub }:
+
+rustPlatform.buildRustPackage rec {
+  pname = "licensor";
+  version = "2.0.0";
+
+  src = fetchFromGitHub {
+    owner = "raftario";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "0bb6q3jpzdygjcs3apl38zzmgkn22ya5wxlqgmlp0cybqbhpi20s";
+  };
+
+  cargoSha256 = "1cvwyj2043vi5905n5126ikwbs3flfgzqkzjnzha0h8in8p3skv1";
+
+  meta = with lib; {
+    description = "Write licenses to stdout";
+    homepage = "https://github.com/raftario/licensor";
+    license = licenses.mit;
+    maintainers = with maintainers; [ filalex77 ];
+    platforms = platforms.all;
+  };
+}
diff --git a/pkgs/development/python-modules/circus/default.nix b/pkgs/tools/networking/circus/default.nix
index a4f96ccaf68e..10edc92a9951 100644
--- a/pkgs/development/python-modules/circus/default.nix
+++ b/pkgs/tools/networking/circus/default.nix
@@ -1,7 +1,10 @@
-{ stdenv, buildPythonPackage, fetchPypi
-, iowait, psutil, pyzmq, tornado, mock }:
+{ stdenv, python3Packages }:
 
-buildPythonPackage rec {
+let
+  inherit (python3Packages) buildPythonApplication fetchPypi iowait psutil pyzmq tornado_4 mock;
+in
+
+buildPythonApplication rec {
   pname = "circus";
   version = "0.15.0";
 
@@ -13,15 +16,14 @@ buildPythonPackage rec {
   postPatch = ''
     # relax version restrictions to fix build
     substituteInPlace setup.py \
-      --replace "pyzmq>=13.1.0,<17.0" "pyzmq>13.1.0" \
-      --replace "tornado>=3.0,<5.0" "tornado>=3.0"
+      --replace "pyzmq>=13.1.0,<17.0" "pyzmq>13.1.0"
   '';
 
   checkInputs = [ mock ];
 
   doCheck = false; # weird error
 
-  propagatedBuildInputs = [ iowait psutil pyzmq tornado ];
+  propagatedBuildInputs = [ iowait psutil pyzmq tornado_4 ];
 
   meta = with stdenv.lib; {
     description = "A process and socket manager";
diff --git a/pkgs/tools/networking/twa/default.nix b/pkgs/tools/networking/twa/default.nix
index fe75b35eea2b..b15b6dcba31e 100644
--- a/pkgs/tools/networking/twa/default.nix
+++ b/pkgs/tools/networking/twa/default.nix
@@ -4,6 +4,7 @@
 , fetchFromGitHub
 , gawk
 , host
+, jq
 , lib
 , makeWrapper
 , ncurses
@@ -12,18 +13,25 @@
 
 stdenv.mkDerivation rec {
   pname = "twa";
-  version = "1.8.0";
+  version = "1.9.1";
 
   src = fetchFromGitHub {
     owner = "trailofbits";
     repo = "twa";
     rev = version;
-    sha256 = "1xq35xyz5536nwrwmlp1wqa7q9jgh90ly7vdy3a5rvxnksx0b2l5";
+    sha256 = "1ab3bcyhfach9y15w8ffvqqan2qk8h62n6z8nqbgygi7n1mf6jzx";
   };
 
   dontBuild = true;
 
-  buildInputs = [ makeWrapper bash gawk curl netcat host.dnsutils ];
+  nativeBuildInputs = [ makeWrapper ];
+
+  buildInputs = [ bash
+                  curl
+                  gawk
+                  host.dnsutils
+                  jq
+                  netcat ];
 
   installPhase = ''
     install -Dm 0755 twa "$out/bin/twa"
@@ -32,7 +40,11 @@ stdenv.mkDerivation rec {
     install -Dm 0644 README.md "$out/share/doc/twa/README.md"
 
     wrapProgram "$out/bin/twa" \
-      --prefix PATH : ${stdenv.lib.makeBinPath [ curl netcat ncurses host.dnsutils ]}
+      --prefix PATH : ${stdenv.lib.makeBinPath [ curl
+                                                 host.dnsutils
+                                                 jq
+                                                 ncurses
+                                                 netcat ]}
   '';
 
   meta = with lib; {
diff --git a/pkgs/tools/package-management/home-manager/default.nix b/pkgs/tools/package-management/home-manager/default.nix
index 5f4cc58c6a22..da4bb43066cc 100644
--- a/pkgs/tools/package-management/home-manager/default.nix
+++ b/pkgs/tools/package-management/home-manager/default.nix
@@ -6,13 +6,13 @@
 stdenv.mkDerivation rec {
 
   pname = "home-manager";
-  version = "2019-10-23";
+  version = "2019-10-29";
 
   src = fetchFromGitHub {
     owner = "rycee";
     repo = "home-manager";
-    rev = "1b987952b5f7d18f0bb66317cf18ffda43ad45aa";
-    sha256 = "1jdmxdnyd6jaiqjjkzw3qr0ia4qvmwmgfn05hbph37v03p55ah5q";
+    rev = "450571056552c9311fcb2894328696b535265593";
+    sha256 = "1rlv234m0bqj9x2y9wnl8z3yq8mixzq8332nqlb8fw9k8mazis6s";
   };
 
   nativeBuildInputs = [ makeWrapper ];
diff --git a/pkgs/tools/security/fierce/default.nix b/pkgs/tools/security/fierce/default.nix
index 809d8e29ea35..5d12a0008158 100644
--- a/pkgs/tools/security/fierce/default.nix
+++ b/pkgs/tools/security/fierce/default.nix
@@ -2,13 +2,13 @@
 
 python3.pkgs.buildPythonApplication rec {
   pname = "fierce";
-  version = "1.3.0";
+  version = "1.4.0";
 
   src = fetchFromGitHub {
     owner = "mschwager";
     repo = pname;
     rev = version;
-    sha256 = "0cdp9rpabazyfnks30rsf3qfdi40z1bkspxk4ds9bm82kpq33jxy";
+    sha256 = "11yaz8ap9swx95j3wpqh0b6jhw6spqgfnsyn1liw9zqi4jwgiax7";
   };
 
   propagatedBuildInputs = [ python3.pkgs.dns ];
diff --git a/pkgs/tools/security/trousers/default.nix b/pkgs/tools/security/trousers/default.nix
index 6f7fed20c89d..a38c011d7a49 100644
--- a/pkgs/tools/security/trousers/default.nix
+++ b/pkgs/tools/security/trousers/default.nix
@@ -2,13 +2,15 @@
 
 stdenv.mkDerivation rec {
   pname = "trousers";
-  version = "0.3.13";
+  version = "0.3.14";
 
   src = fetchurl {
     url = "mirror://sourceforge/trousers/trousers/${version}/${pname}-${version}.tar.gz";
-    sha256 = "1lvnla1c1ig2w3xvvrqg2w9qm7a1ygzy1j2gg8j7p8c87i58x45v";
+    sha256 = "0iwgsbrbb7nfqgl61x8aailwxm8akxh9gkcwxhsvf50x4qx72l6f";
   };
 
+  sourceRoot = ".";
+
   nativeBuildInputs = [ pkgconfig ];
   buildInputs = [ openssl ];
 
@@ -16,21 +18,14 @@ stdenv.mkDerivation rec {
 
   configureFlags = [ "--disable-usercheck" ];
 
-  # Attempt to remove -std=gnu89 when updating if using gcc5
-  NIX_CFLAGS_COMPILE = "-std=gnu89 -DALLOW_NON_TSS_CONFIG_FILE";
-  NIX_LDFLAGS = "-lgcc_s";
-
-  # Fix broken libtool file
-  preFixup = stdenv.lib.optionalString (!stdenv.isDarwin) ''
-    sed 's,-lcrypto,-L${openssl.out}/lib -lcrypto,' -i $out/lib/libtspi.la
-  '';
+  NIX_CFLAGS_COMPILE = [ "-DALLOW_NON_TSS_CONFIG_FILE" ];
+  enableParallelBuilding = true;
 
   meta = with stdenv.lib; {
     description = "Trusted computing software stack";
     homepage    = http://trousers.sourceforge.net/;
-    license     = licenses.cpl10;
+    license     = licenses.bsd3;
     maintainers = [ maintainers.ak ];
     platforms   = platforms.linux;
   };
 }
-
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 0a87dc27f936..9756a00b9b94 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2311,6 +2311,8 @@ in
     buildGoModule = buildGo112Module;
   };
 
+  circus = callPackage ../tools/networking/circus { };
+
   # Cleanup before 20.03:
   citrix_receiver = throw "citrix_receiver has been discontinued by Citrix (https://docs.citrix.com/en-us/citrix-workspace-app.html). Please use citrix_workspace.";
   citrix_receiver_13_10_0 = citrix_receiver;
@@ -2969,6 +2971,7 @@ in
   evemu = callPackage ../tools/system/evemu { };
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
+  # When updating make sure to update all plugins or they will break!
   elk6Version = "6.8.3";
   elk7Version = "7.3.1";
 
@@ -4388,6 +4391,8 @@ in
 
   libsmi = callPackage ../development/libraries/libsmi { };
 
+  licensor = callPackage ../tools/misc/licensor { };
+
   lesspipe = callPackage ../tools/misc/lesspipe { };
 
   liquidsoap = callPackage ../tools/audio/liquidsoap/full.nix {
@@ -6757,9 +6762,7 @@ in
 
   trilium = callPackage ../applications/office/trilium { };
 
-  trousers = callPackage ../tools/security/trousers {
-    openssl = openssl_1_0_2;
-  };
+  trousers = callPackage ../tools/security/trousers { };
 
   trx = callPackage ../tools/audio/trx { };
 
@@ -25337,9 +25340,7 @@ in
 
   zimg = callPackage ../development/libraries/zimg { };
 
-  wtf = callPackage ../applications/misc/wtf {
-    buildGoModule = buildGo112Module;
-  };
+  wtf = callPackage ../applications/misc/wtf { };
 
   zk-shell = callPackage ../applications/misc/zk-shell { };
 
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index b1a402a62991..2321821a536e 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -1643,8 +1643,6 @@ in {
 
   cmarkgfm = callPackage ../development/python-modules/cmarkgfm { };
 
-  circus = callPackage ../development/python-modules/circus {};
-
   colorcet = callPackage ../development/python-modules/colorcet { };
 
   coloredlogs = callPackage ../development/python-modules/coloredlogs { };
@@ -1964,6 +1962,8 @@ in {
 
   impacket = callPackage ../development/python-modules/impacket { };
 
+  jsonlines = callPackage ../development/python-modules/jsonlines { };
+
   jsonrpc-async = callPackage ../development/python-modules/jsonrpc-async { };
 
   jsonrpc-base = callPackage ../development/python-modules/jsonrpc-base { };
@@ -4689,6 +4689,8 @@ in {
 
   pyhocon = callPackage ../development/python-modules/pyhocon { };
 
+  pyjson5 = callPackage ../development/python-modules/pyjson5 {};
+
   pymaging = callPackage ../development/python-modules/pymaging { };
 
   pymaging_png = callPackage ../development/python-modules/pymaging_png { };