Merge #316074: libiconv-darwin: fix crash and add support for static builds

...into staging-24.05

7 files changed, 325 insertions, 29 deletions

diff --git a/pkgs/by-name/li/libiconv-darwin/0001-Support-static-module-loading.patch b/pkgs/by-name/li/libiconv-darwin/0001-Support-static-module-loading.patch
new file mode 100644
index 000000000000..691b483b27ef
--- /dev/null
+++ b/pkgs/by-name/li/libiconv-darwin/0001-Support-static-module-loading.patch
@@ -0,0 +1,72 @@
+From a3e945c630180e9aa182603207238302e58fe8fe Mon Sep 17 00:00:00 2001
+From: Randy Eckenrode <randy@largeandhighquality.com>
+Date: Sat, 25 May 2024 19:03:58 -0400
+Subject: [PATCH 1/2] Support static module loading
+
+---
+ citrus/citrus_module.c | 21 ++++++++++++++++++---
+ 1 file changed, 18 insertions(+), 3 deletions(-)
+
+diff --git a/citrus/citrus_module.c b/citrus/citrus_module.c
+index 5103d30..c22c9da 100644
+--- a/citrus/citrus_module.c
++++ b/citrus/citrus_module.c
+@@ -324,22 +324,36 @@ out:
+ 	return (path[0] ? path : NULL);
+ }
+ 
++#if defined(ENABLE_STATIC)
++#include "static-modules.h"
++#endif
++
+ void *
+ _citrus_find_getops(_citrus_module_t handle, const char *modname,
+     const char *ifname)
+ {
+ 	char name[PATH_MAX];
+ 	void *p;
+-
++#if defined(ENABLE_STATIC)
++	const struct getops_pair* res = lookup_getops(modname, strlen(modname));
++	p = res ? res->opsfn : NULL;
++#else
+ 	snprintf(name, sizeof(name), "_citrus_%s_%s_getops",
+ 	    modname, ifname);
+ 	p = dlsym((void *)handle, name);
++#endif
+ 	return (p);
+ }
+ 
+ int
+ _citrus_load_module(_citrus_module_t *rhandle, const char *encname)
+ {
++#if defined(ENABLE_STATIC)
++	if (is_known_encoding(encname, strnlen(encname, MAX_WORD_LENGTH)) > MAX_HASH_VALUE) {
++		return (EINVAL);
++	}
++	*rhandle = (_citrus_module_t)encodings;
++#else
+ 	const char *p;
+ 	char path[PATH_MAX];
+ 	void *handle;
+@@ -368,14 +382,15 @@ _citrus_load_module(_citrus_module_t *rhandle, const char *encname)
+ 	}
+ 
+ 	*rhandle = (_citrus_module_t)handle;
+-
++#endif
+ 	return (0);
+ }
+ 
+ void
+ _citrus_unload_module(_citrus_module_t handle)
+ {
+-
++#if !defined(ENABLE_STATIC)
+ 	if (handle)
+ 		dlclose((void *)handle);
++#endif
+ }
+-- 
+2.44.1
+
diff --git a/pkgs/by-name/li/libiconv-darwin/0002-Fix-ISO-2022-out-of-bounds-write-with-encoded-charac.patch b/pkgs/by-name/li/libiconv-darwin/0002-Fix-ISO-2022-out-of-bounds-write-with-encoded-charac.patch
new file mode 100644
index 000000000000..e6bce71abd93
--- /dev/null
+++ b/pkgs/by-name/li/libiconv-darwin/0002-Fix-ISO-2022-out-of-bounds-write-with-encoded-charac.patch
@@ -0,0 +1,43 @@
+From 6a2c81d23558d19a68d5494f8f8618bd55c89405 Mon Sep 17 00:00:00 2001
+From: Randy Eckenrode <randy@largeandhighquality.com>
+Date: Mon, 27 May 2024 13:43:43 -0400
+Subject: [PATCH 2/2] Fix ISO-2022 out-of-bounds write with encoded characters
+
+---
+ libiconv_modules/ISO2022/citrus_iso2022.c | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/libiconv_modules/ISO2022/citrus_iso2022.c b/libiconv_modules/ISO2022/citrus_iso2022.c
+index 46da1d6..c2eeaa8 100644
+--- a/libiconv_modules/ISO2022/citrus_iso2022.c
++++ b/libiconv_modules/ISO2022/citrus_iso2022.c
+@@ -1031,7 +1031,7 @@ _ISO2022_sputwchar(_ISO2022EncodingInfo * __restrict ei, wchar_t wc,
+ {
+ 	_ISO2022Charset cs;
+ 	char *p;
+-	char tmp[MB_LEN_MAX];
++	char tmp[MB_LEN_MAX + 4];
+ 	size_t len;
+ 	int bit8, i = 0, target;
+ 	unsigned char mask;
+@@ -1196,7 +1196,7 @@ _citrus_ISO2022_put_state_reset(_ISO2022EncodingInfo * __restrict ei,
+     size_t * __restrict nresult)
+ {
+ 	char *result;
+-	char buf[MB_LEN_MAX];
++	char buf[MB_LEN_MAX + 4];
+ 	size_t len;
+ 	int ret;
+ 
+@@ -1225,7 +1225,7 @@ _citrus_ISO2022_wcrtomb_priv(_ISO2022EncodingInfo * __restrict ei,
+     _ISO2022State * __restrict psenc, size_t * __restrict nresult)
+ {
+ 	char *result;
+-	char buf[MB_LEN_MAX];
++	char buf[MB_LEN_MAX + 4];
+ 	size_t len;
+ 	int ret;
+ 
+-- 
+2.44.1
+
diff --git a/pkgs/by-name/li/libiconv-darwin/meson.build b/pkgs/by-name/li/libiconv-darwin/meson.build
index 99adeb7fe084..9473cb891142 100644
--- a/pkgs/by-name/li/libiconv-darwin/meson.build
+++ b/pkgs/by-name/li/libiconv-darwin/meson.build
@@ -19,6 +19,19 @@ i18nmoduledir = prefix_libdir / 'i18n'
 esdbdir = prefix_datadir / 'i18n/esdb'
 csmapperdir = prefix_datadir / 'i18n/csmapper'
 
+is_static = get_option('default_library') == 'static'
+static_suffix = is_static ? '_static' : ''
+
+# Generators
+if is_static
+    gperf_bin = find_program('gperf', required : true)
+    gperf = generator(
+        gperf_bin,
+        arguments : ['@INPUT@', '--output-file=@OUTPUT@'],
+        output : '@BASENAME@.h'
+    )
+endif
+
 
 # Libraries
 libcharset = library(
@@ -37,15 +50,16 @@ install_headers(
 )
 
 libiconv = library(
-    'iconv',
+    'iconv' + static_suffix,
     build_rpath : fs.parent(libcharset.full_path()),
     c_args : [
         f'-D_PATH_I18NMODULE="@i18nmoduledir@"',
         f'-D_PATH_ESDB="@esdbdir@"',
-        f'-D_PATH_CSMAPPER="@csmapperdir@"'
+        f'-D_PATH_CSMAPPER="@csmapperdir@"',
+        is_static ? '-DENABLE_STATIC=1' : [ ]
     ],
     darwin_versions : '7',
-    install : true,
+    install : not is_static,
     include_directories : ['citrus', 'libcharset'],
     link_args : ['-Wl,-reexport_library', fs.name(libcharset.full_path())],
     link_depends : [libcharset],
@@ -84,6 +98,7 @@ libiconv = library(
         'citrus/iconvctl.c',
         'citrus/iconvlist.c',
         'citrus/iconv.c',
+        is_static ? gperf.process('static-modules.gperf') : [ ]
     ],
     soversion : '2'
 )
@@ -99,19 +114,6 @@ install_man(
 )
 
 
-# Binaries
-executable(
-    'iconv',
-    install : true,
-    include_directories : ['citrus', 'libcharset'],
-    link_with : [libiconv],
-    sources : [
-        'iconv/iconv.c'
-    ]
-)
-install_man('iconv/iconv.1')
-
-
 # Data
 ## csmapper
 csmapper_modules = [
@@ -216,6 +218,7 @@ libiconv_modules = [
     'mapper_zone'
 ]
 
+modules = [ ]
 foreach module : libiconv_modules
     module_source = module.to_lower()
     module_path = 'libiconv_modules' / module
@@ -230,14 +233,19 @@ foreach module : libiconv_modules
     # See: https://github.com/apple-oss-distributions/libiconv/blob/81be60a93521c931a01aab9c747dd2b078bc0679/libiconv.xcodeproj/project.pbxproj#L2549-L2556
     # See also: https://cgit.freebsd.org/src/tree/lib/libiconv_modules/mapper_parallel/Makefile?id=9241ebc796c11cf133c550f188f324bd2c12d89a
     if module == 'mapper_parallel'
+        # Skip including mapper_parallel since it’s the same as mapper_serial, which would result in duplicate symbols.
+        if is_static
+            continue
+        endif
+
         module_source = 'mapper_serial'
         module_path = 'libiconv_modules/mapper_serial'
     endif
 
-    library(
+    modules += library(
         module,
         darwin_versions : '1',
-        install : true,
+        install : not is_static,
         install_dir : i18nmoduledir,
         include_directories : [module_path, 'citrus', 'libcharset'] + extra_headers,
         link_with : [libiconv],
@@ -249,6 +257,38 @@ foreach module : libiconv_modules
 endforeach
 
 
+# Bundle the modules into libiconv.a
+if is_static
+    objects = [libiconv.extract_all_objects(recursive : true)]
+    foreach module : modules
+        objects += module.extract_all_objects(recursive : true)
+    endforeach
+
+    libiconv = static_library(
+        'iconv',
+        install : true,
+        objects : objects
+    )
+endif
+
+
+# Binaries
+executable(
+    'iconv',
+    install : true,
+    include_directories : ['citrus', 'libcharset'],
+    link_with : [
+        libiconv,
+        # Darwin’s system `iconv` relies on the reexported symbols from libiconv
+        is_static ? libcharset : [ ]
+    ],
+    sources : [
+        'iconv/iconv.c'
+    ]
+)
+install_man('iconv/iconv.1')
+
+
 # Tests
 if get_option('tests') == true
 ## Only required for running the tests
@@ -259,7 +299,11 @@ if get_option('tests') == true
             suite,
             dependencies : [atf],
             include_directories : ['citrus', 'libcharset'],
-            link_with : [libiconv],
+            link_with : [
+                libiconv,
+                # Make sure the libcharset reexport is working by relying on libiconv to provide its symbols.
+                is_static ? libcharset : [ ]
+            ],
             sources : [test_src]
         )
 
diff --git a/pkgs/by-name/li/libiconv-darwin/nixpkgs_test.c b/pkgs/by-name/li/libiconv-darwin/nixpkgs_test.c
index 40400ef83283..90981c5820f3 100644
--- a/pkgs/by-name/li/libiconv-darwin/nixpkgs_test.c
+++ b/pkgs/by-name/li/libiconv-darwin/nixpkgs_test.c
@@ -73,10 +73,44 @@ ATF_TC_BODY(test_cp932_cp932, tc)
     ATF_CHECK_STREQ(expected, output);
 }
 
+ATF_TC(test_iso2022_crash);
+ATF_TC_HEAD(test_iso2022_crash, tc)
+{
+    atf_tc_set_md_var(tc, "descr", "regression test for converting to ISO-2022 with escape sequences");
+}
+ATF_TC_BODY(test_iso2022_crash, tc)
+{
+    char expected[] = "";
+    size_t expected_length = sizeof(expected) - 1;
+
+    char input[] = "\x41\x41\x41\x41\x41\xe5\x8a\x84";
+    size_t input_length = sizeof(input) - 1;
+
+    size_t output_available = sizeof(expected) - 1 ;
+    char output[sizeof(expected)] = { 0 };
+
+    iconv_t cd = iconv_open("ISO-2022-CN-EXT", "UTF-8");
+    ATF_REQUIRE((size_t)cd != -1);
+
+    char* input_buf = input;
+    char* output_buf = output;
+
+    size_t res = iconv(cd, &input_buf, &input_length, &output_buf, &output_available);
+    iconv_close(cd);
+
+    ATF_CHECK(res == -1);
+
+    size_t output_length = sizeof(output) - output_available - 1;
+
+    ATF_CHECK_INTEQ(expected_length, output_length);
+    ATF_CHECK_STREQ(expected, output);
+}
+
 ATF_TP_ADD_TCS(tp)
 {
     ATF_TP_ADD_TC(tp, test_cp932_eucjp);
     ATF_TP_ADD_TC(tp, test_cp932_cp932);
+    ATF_TP_ADD_TC(tp, test_iso2022_crash);
 
     return atf_no_error();
 }
diff --git a/pkgs/by-name/li/libiconv-darwin/package.nix b/pkgs/by-name/li/libiconv-darwin/package.nix
index bb5291831b65..0eb6c9f08962 100644
--- a/pkgs/by-name/li/libiconv-darwin/package.nix
+++ b/pkgs/by-name/li/libiconv-darwin/package.nix
@@ -4,6 +4,7 @@
   fetchFromGitHub,
   fetchpatch,
   atf,
+  gperf,
   libiconvReal,
   meson,
   ninja,
@@ -11,6 +12,9 @@
   gitUpdater,
 }:
 
+let
+  inherit (stdenv) hostPlatform;
+in
 stdenv.mkDerivation (finalAttrs: {
   pname = "libiconv";
   version = "99";
@@ -20,6 +24,9 @@ stdenv.mkDerivation (finalAttrs: {
     "dev"
   ];
 
+  # Propagate `out` only when there are dylibs to link (i.e., don’t propagate when doing a static build).
+  propagatedBuildOutputs = lib.optionalString (!hostPlatform.isStatic) "out";
+
   src = fetchFromGitHub {
     owner = "apple-oss-distributions";
     repo = "libiconv";
@@ -27,7 +34,13 @@ stdenv.mkDerivation (finalAttrs: {
     hash = "sha256-TGt6rsU52ztfW2rCqwnhMAExLbexI/59IoDOGY+XGu0=";
   };
 
-  inherit (libiconvReal) setupHooks;
+  setupHooks =
+    libiconvReal.setupHooks
+    ++ lib.optionals hostPlatform.isStatic [ ./static-setup-hook.sh ];
+
+  patches = lib.optionals hostPlatform.isStatic [ ./0001-Support-static-module-loading.patch ] ++ [
+    ./0002-Fix-ISO-2022-out-of-bounds-write-with-encoded-charac.patch
+  ];
 
   postPatch =
     ''
@@ -42,30 +55,42 @@ stdenv.mkDerivation (finalAttrs: {
       header
 
       cp ${./nixpkgs_test.c} tests/libiconv/nixpkgs_test.c
+    ''
+    + lib.optionalString hostPlatform.isStatic ''
+      cp ${./static-modules.gperf} static-modules.gperf
     '';
 
   strictDeps = true;
 
-  nativeBuildInputs = [
-    meson
-    ninja
-  ];
+  nativeBuildInputs =
+    [
+      meson
+      ninja
+    ]
+    # Dynamic builds use `dlopen` to load modules, but static builds have to link them all.
+    # `gperf` is used to generate a lookup table from module to ops functions.
+    ++ lib.optionals hostPlatform.isStatic [ gperf ];
 
   mesonBuildType = "release";
 
   mesonFlags = [ (lib.mesonBool "tests" finalAttrs.doInstallCheck) ];
 
-  postInstall = lib.optionalString stdenv.isDarwin ''
-    ${stdenv.cc.targetPrefix}install_name_tool "$out/lib/libiconv.2.dylib" \
-      -change '@rpath/libcharset.1.dylib' "$out/lib/libcharset.1.dylib"
-  '';
+  postInstall =
+    lib.optionalString (stdenv.isDarwin && !hostPlatform.isStatic) ''
+      ${stdenv.cc.targetPrefix}install_name_tool "$out/lib/libiconv.2.dylib" \
+        -change '@rpath/libcharset.1.dylib' "$out/lib/libcharset.1.dylib"
+    ''
+    # Move the static library to the `dev` output
+    + lib.optionalString hostPlatform.isStatic ''
+      moveToOutput lib "$dev"
+    '';
 
   # Tests have to be run in `installCheckPhase` because libiconv expects to `dlopen`
   # modules from `$out/lib/i18n`.
   nativeInstallCheckInputs = [ pkg-config ];
   installCheckInputs = [ atf ];
 
-  doInstallCheck = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+  doInstallCheck = stdenv.buildPlatform.canExecute hostPlatform;
 
   # Can’t use `mesonCheckPhase` because it runs the wrong hooks for `installCheckPhase`.
   installCheckPhase = ''
diff --git a/pkgs/by-name/li/libiconv-darwin/static-modules.gperf b/pkgs/by-name/li/libiconv-darwin/static-modules.gperf
new file mode 100644
index 000000000000..be916fb257ec
--- /dev/null
+++ b/pkgs/by-name/li/libiconv-darwin/static-modules.gperf
@@ -0,0 +1,73 @@
+%language=ANSI-C
+%compare-strncmp
+%pic
+%readonly-tables
+%{
+#include "citrus_namespace.h"
+#include "citrus_types.h"
+#include "citrus_module.h"
+#include "citrus_hash.h"
+
+#include "citrus_iconv.h"
+#include "citrus_mapper.h"
+#include "citrus_stdenc.h"
+
+extern _CITRUS_STDENC_GETOPS_FUNC(BIG5);
+extern _CITRUS_STDENC_GETOPS_FUNC(DECHanyu);
+extern _CITRUS_STDENC_GETOPS_FUNC(DECKanji);
+extern _CITRUS_STDENC_GETOPS_FUNC(EUC);
+extern _CITRUS_STDENC_GETOPS_FUNC(EUCTW);
+extern _CITRUS_STDENC_GETOPS_FUNC(GBK2K);
+extern _CITRUS_STDENC_GETOPS_FUNC(HZ);
+extern _CITRUS_STDENC_GETOPS_FUNC(ISO2022);
+extern _CITRUS_STDENC_GETOPS_FUNC(JOHAB);
+extern _CITRUS_STDENC_GETOPS_FUNC(MSKanji);
+extern _CITRUS_STDENC_GETOPS_FUNC(UES);
+extern _CITRUS_STDENC_GETOPS_FUNC(UTF1632);
+extern _CITRUS_STDENC_GETOPS_FUNC(UTF7);
+extern _CITRUS_STDENC_GETOPS_FUNC(UTF8);
+extern _CITRUS_STDENC_GETOPS_FUNC(UTF8MAC);
+extern _CITRUS_STDENC_GETOPS_FUNC(VIQR);
+extern _CITRUS_STDENC_GETOPS_FUNC(ZW);
+
+extern _CITRUS_ICONV_GETOPS_FUNC(iconv_none);
+extern _CITRUS_ICONV_GETOPS_FUNC(iconv_std);
+
+extern _CITRUS_MAPPER_GETOPS_FUNC(mapper_646);
+extern _CITRUS_MAPPER_GETOPS_FUNC(mapper_none);
+extern _CITRUS_MAPPER_GETOPS_FUNC(mapper_parallel);
+extern _CITRUS_MAPPER_GETOPS_FUNC(mapper_serial);
+extern _CITRUS_MAPPER_GETOPS_FUNC(mapper_std);
+extern _CITRUS_MAPPER_GETOPS_FUNC(mapper_zone);
+%}
+%define lookup-function-name lookup_getops
+%define hash-function-name is_known_encoding
+%define string-pool-name encodings
+%struct-type
+struct getops_pair { int name; void* opsfn; };
+%%
+BIG5, _citrus_BIG5_stdenc_getops
+DECHanyu, _citrus_DECHanyu_stdenc_getops
+DECKanji, _citrus_DECKanji_stdenc_getops
+EUC, _citrus_EUC_stdenc_getops
+EUCTW, _citrus_EUCTW_stdenc_getops
+GBK2K, _citrus_GBK2K_stdenc_getops
+HZ, _citrus_HZ_stdenc_getops
+ISO2022, _citrus_ISO2022_stdenc_getops
+JOHAB, _citrus_JOHAB_stdenc_getops
+MSKanji, _citrus_MSKanji_stdenc_getops
+UES, _citrus_UES_stdenc_getops
+UTF1632, _citrus_UTF1632_stdenc_getops
+UTF7, _citrus_UTF7_stdenc_getops
+UTF8, _citrus_UTF8_stdenc_getops
+UTF8MAC, _citrus_UTF8MAC_stdenc_getops
+VIQR, _citrus_VIQR_stdenc_getops
+ZW, _citrus_ZW_stdenc_getops
+iconv_none, _citrus_iconv_none_iconv_getops
+iconv_std, _citrus_iconv_std_iconv_getops
+mapper_646, _citrus_mapper_646_mapper_getops
+mapper_none, _citrus_mapper_none_mapper_getops
+mapper_serial, _citrus_mapper_serial_mapper_getops
+mapper_parallel, _citrus_mapper_parallel_mapper_getops
+mapper_std, _citrus_mapper_std_mapper_getops
+mapper_zone, _citrus_mapper_zone_mapper_getops
diff --git a/pkgs/by-name/li/libiconv-darwin/static-setup-hook.sh b/pkgs/by-name/li/libiconv-darwin/static-setup-hook.sh
new file mode 100644
index 000000000000..2ad108b82f1d
--- /dev/null
+++ b/pkgs/by-name/li/libiconv-darwin/static-setup-hook.sh
@@ -0,0 +1,5 @@
+# Darwin’s dynamic libiconv reexports libcharset, so reproduce that in static builds.
+if [ -z "${dontAddExtraLibs-}" ]; then
+    getHostRole
+    export NIX_LDFLAGS${role_post}+=" -lcharset"
+fi