Skip to content

Commit 46167af

Browse files
committed
Use GNU libiconv instead of system iconv
1 parent 46d19f9 commit 46167af

13 files changed

+79
-59
lines changed

.github/workflows/c-cpp.yml

+7-6
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
release: false
3232
msystem: ${{matrix.sys}}
3333
install: >-
34-
patch git groff help2man rsync texinfo flex
34+
patch git groff help2man rsync texinfo flex gperf
3535
mingw-w64-${{matrix.arch}}-autotools
3636
mingw-w64-${{matrix.arch}}-gcc
3737
mingw-w64-${{matrix.arch}}-python3
@@ -42,17 +42,18 @@ jobs:
4242
submodules: true
4343
- name: Install dependencies (Ubuntu)
4444
if: ${{ matrix.os == 'ubuntu-latest' }}
45-
run: sudo apt-get -y install python3 cython3 flex help2man autopoint texinfo gettext
45+
run: sudo apt-get -y install python3 cython3 flex help2man autopoint texinfo gettext gperf groff
4646
- name: Install dependencies (macOS)
4747
if: ${{ matrix.os == 'macos-latest' }}
4848
run: |
49-
brew install autoconf automake help2man python cython flex texinfo
49+
brew install autoconf automake help2man python cython flex texinfo gperf groff gnu-tar
5050
# Prepend optional brew binary directories to PATH
51-
echo "/usr/local/opt/flex/bin:/usr/local/opt/m4/bin:/usr/local/opt/texinfo/bin:/usr/local/opt/gettext/bin:/usr/local/opt/cython/bin" >> $GITHUB_PATH
51+
echo "/usr/local/opt/gnu-tar/libexec/gnubin:/usr/local/opt/flex/bin:/usr/local/opt/m4/bin:/usr/local/opt/texinfo/bin:/usr/local/opt/gettext/bin:/usr/local/opt/cython/bin" >> $GITHUB_PATH
5252
- name: Set up environment (Ubuntu)
5353
if: ${{ matrix.os == 'ubuntu-latest' }}
54+
# FIXME: ASAN
5455
run: |
55-
echo "ASAN=yes" >> $GITHUB_ENV
56+
echo "# ASAN=yes" >> $GITHUB_ENV
5657
echo "LSAN_OPTIONS=verbosity=1:log_threads=1" >> $GITHUB_ENV
5758
- name: Set up environment (macOS)
5859
if: ${{ matrix.os == 'macos-latest' }}
@@ -68,6 +69,6 @@ jobs:
6869
- name: Build
6970
run: |
7071
./bootstrap
71-
if [[ "$ASAN" == "yes" ]]; then ./configure --enable-silent-rules CFLAGS="-g3 -fsanitize=address -fsanitize=undefined" LDFLAGS="-fsanitize=address -fsanitize=undefined" TESTS_ENVIRONMENT_EXTRA="LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so PYTHONMALLOC=malloc"; else ./configure --enable-silent-rules; fi
72+
if [[ "$ASAN" == "yes" ]]; then ./configure --enable-silent-rules CFLAGS="-g3 -fPIC -fsanitize=address -fsanitize=undefined" LDFLAGS="-fsanitize=address -fsanitize=undefined" TESTS_ENVIRONMENT_EXTRA="LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so PYTHONMALLOC=malloc"; else ./configure --enable-silent-rules; fi
7273
make check
7374
make distcheck

.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@
44
[submodule "gl-mod/bootstrap"]
55
path = gl-mod/bootstrap
66
url = https://github.com/gnulib-modules/bootstrap.git
7+
[submodule "libiconv"]
8+
path = libiconv
9+
url = https://git.savannah.gnu.org/git/libiconv.git

Makefile.am

+13-3
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,23 @@ AUTOMAKE_OPTIONS = gnits
1919

2020
ACLOCAL_AMFLAGS = -I m4
2121

22-
SUBDIRS = doc lib src po tests contrib
22+
SUBDIRS = libiconv doc lib src po tests contrib
2323

2424
EXTRA_DIST = COPYING-LIB \
2525
m4/gnulib-cache.m4 tables.py \
2626
keld/NomsSeulsfinal.lst keld/charsets.def keld/control.def \
2727
keld/iso10646.def keld/other.def keld/rfc1345.txt
2828

29+
dist-hook:
30+
@if test "$(VPATH)" = ""; then \
31+
cd libiconv && \
32+
make dist && \
33+
cd ../$(PACKAGE)-$(VERSION) && \
34+
tar -z -x -f ../libiconv/libiconv-*.tar.gz --transform="s|^libiconv-[^/]*|libiconv|"; \
35+
else \
36+
echo "I can't make a proper dist tarball in a VPATH build!"; \
37+
fi
38+
2939
release: distcheck
3040
git diff --exit-code && \
3141
git tag -a -m "Release tag" "v$(VERSION)" && \
@@ -37,7 +47,7 @@ release: distcheck
3747
dist_type=tar.gz
3848

3949
# Ignore built files that are part of the distribution (specifically,
40-
# src/recode.1).
50+
# src/recode.1), and the libiconv subdirectory.
4151
distcleancheck_listfiles = \
42-
find . -type f -exec sh -c 'test -f $(srcdir)/$$1 || echo $$1' \
52+
find . -prune libiconv -type f -exec sh -c 'test -f $(srcdir)/$$1 || echo $$1' \
4353
sh '{}' ';'

bootstrap.conf

+22-7
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# bootstrap.conf (Recode) version 2023-02-11
1+
# bootstrap.conf (Recode) version 2023-02-25
22

33
# This file is part of Recode.
44
#
@@ -42,7 +42,10 @@ buildreq='
4242

4343
# Non-default gnulib directory options.
4444
local_gl_path=gl-mod/bootstrap:gl
45-
gnulib_git_submodules=gl-mod/bootstrap
45+
gnulib_git_submodules='
46+
gl-mod/bootstrap
47+
libiconv
48+
'
4649

4750
# Additional gnulib-tool options to use.
4851
gnulib_tool_options='
@@ -58,7 +61,6 @@ gnulib_modules='
5861
getopt-posix
5962
gettext-h
6063
hash
61-
iconv
6264
isatty
6365
localcharset
6466
manywarnings
@@ -87,18 +89,21 @@ gnulib_non_module_files="$gnulib_non_module_files"'
8789
# Copyright holder
8890
copyright_holder="Free Software Foundation, Inc."
8991

92+
# Don't run autoreconf recursively
93+
AUTORECONF="autoreconf --no-recursive"
94+
9095

9196
## --------------- ##
9297
## Hook functions. ##
9398
## --------------- ##
9499

95-
# enchant_ignore_gnulib_ignore
96-
# -------------------------
100+
# recode_ignore_gnulib_ignore
101+
# ---------------------------
97102
# gnulib-tool updates m4/.gitignore and lib/.gitignore, and keeping
98103
# generated files under version control does not make sense. Since
99104
# lib is entirely ignored, we only need to prepopulate the m4 ignore
100105
# files with generated files not tracked by gnulib-tool.
101-
enchant_ignore_gnulib_ignore ()
106+
recode_ignore_gnulib_ignore ()
102107
{
103108
$debug_cmd
104109

@@ -116,7 +121,17 @@ gnulib-comp.m4
116121
EOF
117122
fi
118123
}
119-
func_add_hook func_prep enchant_ignore_gnulib_ignore
124+
func_add_hook func_prep recode_ignore_gnulib_ignore
125+
126+
# recode_autogen_libiconv
127+
# -----------------------
128+
recode_autogen_libiconv ()
129+
{
130+
$debug_cmd
131+
132+
( unset MAKEFLAGS; export GNULIB_SRCDIR=$(pwd)/gnulib; cd ./libiconv; ln -sf ../build-aux/libiconv-configure.gnu configure.gnu; ./autogen.sh )
133+
}
134+
func_add_hook func_reconfigure recode_autogen_libiconv
120135

121136

122137
# Local variables:

build-aux/libiconv-configure.gnu

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/sh
2+
./configure --disable-shared --enable-extra-encodings "$@"

configure.ac

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
AC_PREREQ([2.71])
66
AC_INIT([recode],[3.7.14],[[email protected]])
77
AC_CONFIG_SRCDIR(src/recode.c)
8+
AC_CONFIG_SUBDIRS([libiconv])
89
AC_CONFIG_AUX_DIR([build-aux])
910
AM_INIT_AUTOMAKE
1011
AC_CONFIG_MACRO_DIR(m4)
@@ -110,8 +111,8 @@ AM_GNU_GETTEXT_VERSION(0.19)
110111
AM_GNU_GETTEXT(external)
111112
AM_XGETTEXT_OPTION([--from-code=UTF-8])
112113

114+
# Output
113115
AC_CONFIG_HEADERS([config.h])
114-
115116
AC_CONFIG_FILES([Makefile contrib/Makefile doc/Makefile lib/Makefile
116117
po/Makefile.in src/Makefile src/recode.x
117118
tests/Makefile tests/setup.py])

doc/recode.texi

+4-6
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ sets and surface encodings. When this cannot be achieved exactly, it
8181
may get rid of the offending characters or fall back on approximations.
8282
The library recognises or produces more than 300 different character sets
8383
and is able to convert files between almost any pair. Most @w{RFC 1345}
84-
character sets, and all character sets from a pre-installed @code{iconv}
84+
character sets, and all character sets from the include GNU @code{libiconv}
8585
library, are supported.
8686
The @code{recode} program is a handy front-end to the library.
8787

@@ -404,11 +404,9 @@ those of Keld Simonsen and Bruno Haible.
404404

405405
@cindex charsets, overview
406406
Recoding is currently possible between many charsets, the bulk of which
407-
is described by @w{RFC 1345} tables or available in a pre-installed
408-
external @code{iconv} library. @xref{Tabular}, and
409-
@pxref{iconv}@footnote{Because @code{iconv} can vary from system to
410-
system, and is itself a complex tool, it can cause recode to behave in
411-
unexpected ways. Therefore, by default it is only used when a conversion
407+
is described by @w{RFC 1345} tables or available in the included
408+
GNU @code{libiconv} library. @xref{Tabular}, and
409+
@pxref{iconv}@footnote{By default, @code{iconv} is only used when a conversion
412410
would not be possible without it. To request that @code{iconv} be used,
413411
use @code{--prefer-iconv}; see @xref{prefer-iconv}. Conversely, you can
414412
disable it with the @code{-x:} option; see @xref{disable-iconv}.}. The

libiconv

Submodule libiconv added at 4b9c27b

src/Makefile.am

+4-2
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ recode_LDADD = librecode.la
5252
librecode_la_SOURCES = charname.c combine.c fr-charname.c iconv.c \
5353
names.c outer.c recode.c request.c strip-pool.c task.c $(ALL_STEPS) \
5454
$(include_HEADERS) $(noinst_HEADERS) $(H_STEPS)
55-
librecode_la_LDFLAGS = -no-undefined -version-info $(VERSION_INFO) $(LTLIBICONV) $(LTLIBINTL) \
55+
librecode_la_LDFLAGS = -no-undefined -version-info $(VERSION_INFO) $(LTLIBINTL) \
5656
$(LIB_CLOCK_GETTIME) $(LIB_GETRANDOM) $(LIB_HARD_LOCALE) $(LIB_MBRTOWC) $(LIB_SETLOCALE_NULL)
57-
librecode_la_LIBADD = ../lib/libgnu.la libmerged.la
57+
librecode_la_LIBADD = ../lib/libgnu.la libmerged.la ../libiconv/lib/libiconv.la
5858

5959
libmerged_la_SOURCES = merged.c
6060
libmerged_la_CPPFLAGS = $(NON_WARN_CFLAGS)
@@ -165,3 +165,5 @@ strip-pool.c strip-data.c: stamp-strip
165165
stamp-strip: ../tables.py $(MNEMONICS_DS) $(CHARSETS_DEF)
166166
$(TABLES_PY) -C $(srcdir) -p $(MNEMONICS_DS) $(CHARSETS_DEF)
167167
@echo timestamp > $(srcdir)/$@
168+
169+
DISTCLEANFILES = iconv.h libiconv.a libiconv.la

src/iconv.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
#include "config.h"
2222
#include "common.h"
2323
#include "decsteps.h"
24-
#include <iconv.h>
24+
#include "iconv.h"
2525
#include "iconvdecl.h"
2626

2727
/*--------------------------------------.

src/iconv.h

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../libiconv/include/iconv.h.inst

tables.py

+19-31
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/python3
22
# -*- coding: utf-8 -*-
33
# Automatically derive Recode table files from various sources.
4-
# Copyright © 1993-2000 Free Software Foundation, Inc.
4+
# Copyright © 1993-2023 Free Software Foundation, Inc.
55
# François Pinard <[email protected]>, 1993.
66

77
# This program is free software; you can redistribute it and/or modify
@@ -40,7 +40,7 @@
4040
When `-F' and `-n' are used, process Alain's tables.
4141
"""
4242

43-
import re, sys, io
43+
import re, sys, os, io
4444

4545
# Character constants.
4646
REPLACEMENT_CHARACTER = 0xFFFD
@@ -98,7 +98,6 @@ def main(self, *arguments):
9898

9999
# Read all data tables.
100100
if self.directory:
101-
import os
102101
os.chdir(self.directory)
103102
if self.iconv:
104103
self.iconv.digest()
@@ -464,32 +463,22 @@ def digest(self):
464463
canonical[charset.upper()] = charset
465464

466465
# Read in the encodings.def file.
467-
sys.stdout.write("Reading from `iconv -l'\n")
468-
libc = None
469-
import os
466+
sys.stdout.write("Reading from `iconv_no_18n -l'\n")
470467
names = []
471-
for line in os.popen('iconv -l'):
472-
if libc is None:
473-
libc = len(line.split('/')) == 3
474-
if libc:
475-
first, second, empty = line.split('/')
476-
assert empty == '\n', repr(line)
477-
name = second or first
478-
if name not in names:
479-
names.append(name)
480-
self.data.append((name, ()))
481-
else:
482-
aliases = []
483-
for alias in line.split():
484-
if alias in canonical:
485-
alias = canonical[alias]
486-
aliases.append(alias)
487-
self.data.append((aliases[0], aliases[1:]))
468+
for line in os.popen('../libiconv/src/iconv_no_i18n -l'):
469+
aliases = []
470+
for alias in line.split():
471+
if alias in canonical:
472+
alias = canonical[alias]
473+
aliases.append(alias)
474+
self.data.append((aliases[0], aliases[1:]))
488475

489476
def complete(self, french):
490-
def write_charset(format, charset):
491-
write(format % charset)
492-
write(format % (charset + "-translit"))
477+
def write_charset_with_aliases(charset, suffix=''):
478+
write(f' "{charset + suffix}",\n')
479+
for alias in aliases[:-1]:
480+
write(f'\t"{alias + suffix}",\n')
481+
write(f'\t"{aliases[-1] + suffix}", NULL,\n')
493482
if not self.do_sources:
494483
return
495484
write = Output(self.SOURCES).write
@@ -505,12 +494,11 @@ def write_charset(format, charset):
505494
% count)
506495
for charset, aliases in self.data:
507496
if aliases:
508-
write_charset(' "%s",\n', charset)
509-
for alias in aliases[:-1]:
510-
write_charset('\t"%s",\n', alias)
511-
write_charset('\t"%s", NULL,\n', aliases[-1])
497+
write_charset_with_aliases(charset)
498+
write_charset_with_aliases(charset, '-translit')
512499
else:
513-
write_charset(' "%s", NULL,\n', charset)
500+
write(f' "{charset}", NULL,\n')
501+
write(f' "{charset}-translit", NULL,\n')
514502
write(' NULL\n'
515503
' };\n')
516504

tests/setup.py.in

-2
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,8 @@ srcdir = '@srcdir@'
99
top_builddir = '@top_builddir@'
1010
top_srcdir = '@top_srcdir@'
1111
libintl = '@LIBINTL@'
12-
libiconv = '@LIBICONV@'
1312
extra_libs = []
1413
extra_libs.extend(libintl.split())
15-
extra_libs.extend(libiconv.split())
1614

1715
# FIXME: On Mingw, Cython seems to call gcc in such a way that it doesn't understand UNIX paths
1816
if os.name == 'nt':

0 commit comments

Comments
 (0)