Merge pull request #175086 from fabaff/spacy-transformers-fix

python310Packages.spacy-transformers: relax transformers constraint

2 files changed, 63 insertions, 15 deletions

diff --git a/pkgs/development/python-modules/spacy-transformers/default.nix b/pkgs/development/python-modules/spacy-transformers/default.nix
index afc3ffbd25ed..70bc4925ae44 100644
--- a/pkgs/development/python-modules/spacy-transformers/default.nix
+++ b/pkgs/development/python-modules/spacy-transformers/default.nix
@@ -33,6 +33,11 @@ buildPythonPackage rec {
     dataclasses
   ];
 
+  postPatch = ''
+    substituteInPlace setup.cfg \
+      --replace "transformers>=3.4.0,<4.18.0" "transformers>=3.4.0 # ,<4.18.0"
+  '';
+
   # Test fails due to missing arguments for trfs2arrays().
   doCheck = false;
 
diff --git a/pkgs/development/python-modules/transformers/default.nix b/pkgs/development/python-modules/transformers/default.nix
index 1716cd44f8df..adef7a19afcf 100644
--- a/pkgs/development/python-modules/transformers/default.nix
+++ b/pkgs/development/python-modules/transformers/default.nix
@@ -1,5 +1,5 @@
-{ buildPythonPackage
-, lib
+{ lib
+, buildPythonPackage
 , fetchFromGitHub
 , pythonOlder
 , cookiecutter
@@ -10,9 +10,14 @@
 , requests
 , numpy
 , packaging
+, tensorflow
+, sagemaker
+, ftfy
 , protobuf
+, scikit-learn
+, pillow
 , pyyaml
-, sacremoses
+, torch
 , tokenizers
 , tqdm
 }:
@@ -20,42 +25,80 @@
 buildPythonPackage rec {
   pname = "transformers";
   version = "4.19.2";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = pname;
     rev = "refs/tags/v${version}";
-    sha256 = "sha256-9r/1vW7Rhv9+Swxdzu5PTnlQlT8ofJeZamHf5X4ql8w=";
+    hash = "sha256-9r/1vW7Rhv9+Swxdzu5PTnlQlT8ofJeZamHf5X4ql8w=";
   };
 
-  nativeBuildInputs = [ packaging ];
-
   propagatedBuildInputs = [
-    cookiecutter
     filelock
     huggingface-hub
     numpy
     protobuf
+    packaging
     pyyaml
     regex
     requests
-    sacremoses
     tokenizers
     tqdm
-  ] ++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ];
+  ] ++ lib.optionals (pythonOlder "3.8") [
+    importlib-metadata
+  ];
+
+  passthru.optional-dependencies = {
+    ja = [
+      # fugashi
+      # ipadic
+      # unidic_lite
+      # unidic
+    ];
+    sklearn = [
+      scikit-learn
+    ];
+    tf = [
+      tensorflow
+      # onnxconverter-common
+      # tf2onnx
+    ];
+    torch = [
+      torch
+    ];
+    tokenizers = [
+      tokenizers
+    ];
+    modelcreation = [
+      cookiecutter
+    ];
+    sagemaker = [
+      sagemaker
+    ];
+    ftfy = [ ftfy ];
+    onnx = [
+      # onnxconverter-common
+      # tf2onnx
+    ];
+    vision = [
+      pillow
+    ];
+  };
+
 
   # Many tests require internet access.
   doCheck = false;
 
-  postPatch = ''
-    sed -ri 's/tokenizers[=>]=[^"]+/tokenizers/g' setup.py src/transformers/dependency_versions_table.py
-  '';
-
-  pythonImportsCheck = [ "transformers" ];
+  pythonImportsCheck = [
+    "transformers"
+  ];
 
   meta = with lib; {
     homepage = "https://github.com/huggingface/transformers";
-    description = "State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch";
+    description = "Natural Language Processing for TensorFlow 2.0 and PyTorch";
     changelog = "https://github.com/huggingface/transformers/releases/tag/v${version}";
     license = licenses.asl20;
     platforms = platforms.unix;