upgrade to ICU 53.1, fdo#77071 related

plus further work in i18npool to make that build

adapt i18npool to ICU 53 upgrade, fdo#77071

Korean charset collator can't be built from ko_charset.txt because of

"The runtime code decomposes Hangul syllables on the fly, with recursive
processing but without making the Jamo pieces visible for matching. It
does not work with certain types of contextual mappings."
"While handling a Hangul syllable, contractions starting with Jamo L or
V would not see the following Jamo of that syllable." (this is where we
bail out already with the first syllable of ko_charset.txt)
Another condition to fail is described as "A contraction ending with
Jamo L or L+V would require generating Hangul syllables in
addTailComposites() (588 for a Jamo L), or decomposing a following
Hangul syllable on the fly, during contraction matching."

Excluded the file from the build for ICU >=53 and hope that ICU in the
mean time handles Korean collation correctly.

Additionally, ICU 53 took ages (if it would had finished at all) to
build the collator from zh_TW_charset.txt because of the \u#### escaped
notation. Converted the file's content to characters using
http://www.rishida.net/tools/conversion/

Change-Id: I64213214b4870e7077f72b95fee1ddc9782c2b21
Reviewed-on: https://gerrit.libreoffice.org/9204
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Eike Rathke <erack@redhat.com>

configure.ac

@@ -8859,7 +8859,7 @@ SYSTEM_GENBRK=
 SYSTEM_GENCCODE=
 SYSTEM_GENCMN=
 
-ICU_MAJOR=52
+ICU_MAJOR=53
 ICU_MINOR=1
 ICU_RECLASSIFIED_CLOSE_PARENTHESIS="TRUE"
 ICU_RECLASSIFIED_PREPEND_SET_EMPTY="TRUE"

download.lst

@@ -72,7 +72,7 @@ export GLM_TARBALL := bae83fa5dc7f081768daace6e199adc3-glm-0.9.4.6-libreoffice.z
 export HSQLDB_TARBALL := 17410483b5b5f267aa18b7e00b65e6e0-hsqldb_1_8_0.zip
 export HUNSPELL_TARBALL := 3121aaf3e13e5d88dfff13fb4a5f1ab8-hunspell-1.3.2.tar.gz
 export HYPHEN_TARBALL := a2f6010987e1c601274ab5d63b72c944-hyphen-2.8.4.tar.gz
-export ICU_TARBALL := 9e96ed4c1d99c0d14ac03c140f9f346c-icu4c-52_1-src.tgz
+export ICU_TARBALL := b73baa6fbdfef197608d1f69300919b9-icu4c-53_1-src.tgz
 export JFREEREPORT_FLOW_ENGINE_TARBALL := ba2930200c9f019c2d93a8c88c651a0f-flow-engine-0.9.4.zip
 export JFREEREPORT_FLUTE_TARBALL := d8bd5eed178db6e2b18eeed243f85aa8-flute-1.1.6.zip
 export JFREEREPORT_LIBBASE_TARBALL := eeb2c7ddf0d302fba4bfc6e97eac9624-libbase-1.1.6.zip

external/icu/UnpackedTarball_icu.mk

@@ -21,7 +21,6 @@ $(eval $(call gb_UnpackedTarball_add_patches,icu,\
 	external/icu/icu4c-mkdir.patch \
 	external/icu/icu4c-buffer-overflow.patch \
 	external/icu/icu4c-$(if $(filter ANDROID,$(OS)),android,rpath).patch \
-	external/icu/icu4c-ubsan.patch.0 \
 ))
 
 # vim: set noet sw=4 ts=4:

external/icu/icu4c-buffer-overflow.patch

@@ -11,7 +11,7 @@ Index: icu/source/tools/pkgdata/pkgdata.cpp
 ===================================================================
 --- build/icu.orig/source/tools/pkgdata/pkgdata.cpp
 +++ build/icu/source/tools/pkgdata/pkgdata.cpp
-@@ -1980,12 +1980,12 @@
+@@ -2060,12 +2060,12 @@
      const char cmd[] = "icu-config --incpkgdatafile";
  
      /* #1 try the same path where pkgdata was called from. */
@@ -22,7 +22,7 @@ Index: icu/source/tools/pkgdata/pkgdata.cpp
 -          uprv_strncat(cmdBuf, U_FILE_SEP_STRING, 1024);
 +          uprv_strncat(cmdBuf, U_FILE_SEP_STRING, sizeof(cmdBuf)-1-strlen(cmdBuf));
        }
--      uprv_strncat(cmdBuf, cmd, 1024);
+-      uprv_strncat(cmdBuf, cmd, 1023);
 +      uprv_strncat(cmdBuf, cmd, sizeof(cmdBuf)-1-strlen(cmdBuf));
        
        if(verbose) {

external/icu/icu4c-ubsan.patch.0

@@ -1,22 +0,0 @@
---- source/common/ustring.cpp
-+++ source/common/ustring.cpp
-@@ -1486,7 +1486,7 @@
- */
- 
- #define STRING_HASH(TYPE, STR, STRLEN, DEREF) \
--    int32_t hash = 0;                         \
-+    uint32_t hash = 0;                         \
-     const TYPE *p = (const TYPE*) STR;        \
-     if (p != NULL) {                          \
-         int32_t len = (int32_t)(STRLEN);      \
---- source/i18n/ucol_tok.cpp
-+++ source/i18n/ucol_tok.cpp
-@@ -48,7 +48,7 @@
- static int32_t U_CALLCONV
- uhash_hashTokens(const UHashTok k)
- {
--    int32_t hash = 0;
-+    uint32_t hash = 0;
-     //uint32_t key = (uint32_t)k.integer;
-     UColToken *key = (UColToken *)k.pointer;
-     if (key != 0) {

external/icu/icu4c-wchar_t.patch

@@ -6,11 +6,11 @@
  # /EHsc enables exception handling
 -# /Zc:wchar_t makes wchar_t a native type. Required for C++ ABI compatibility.
  # -D_CRT_SECURE_NO_DEPRECATE is needed to quiet warnings about using standard C functions.
--CFLAGS+=/GF /nologo
--CXXFLAGS+=/GF /nologo /EHsc /Zc:wchar_t
-+CFLAGS+=/GF /nologo /Zc:wchar_t-
-+CXXFLAGS+=/GF /nologo /EHsc /Zc:wchar_t-
+-CFLAGS+=-GF -nologo
+-CXXFLAGS+=-GF -nologo -EHsc -Zc:wchar_t
++CFLAGS+=-GF -nologo -Zc:wchar_t-
++CXXFLAGS+=-GF -nologo -EHsc -Zc:wchar_t-
  CPPFLAGS+=-D_CRT_SECURE_NO_DEPRECATE
  DEFS+=-DWIN32 -DCYGWINMSVC
- LDFLAGS+=/nologo
+ LDFLAGS+=-nologo
 

i18npool/CustomTarget_collator.mk

@@ -21,8 +21,10 @@ i18npool_COTXTS := \
         ja_charset.txt \
         ja_phonetic_alphanumeric_first.txt \
         ja_phonetic_alphanumeric_last.txt) \
-    $(call gb_Helper_optional_locale,ko, \
-        ko_charset.txt) \
+	$(if $(i18npool_ICULT53), \
+		$(call gb_Helper_optional_locale,ko, \
+			ko_charset.txt) \
+			) \
     $(call gb_Helper_optional_locale,ku, \
         ku_alphanumeric.txt) \
     $(call gb_Helper_optional_locale,ln, \
@@ -55,7 +57,7 @@ $(i18npool_CODIR)/collator_%.cxx : \
 	$(call gb_Helper_abbreviate_dirs, \
 		$(call gb_Helper_execute,gencoll_rule) $< $@ $*)
 
-$(i18npool_CODIR)/lrl_include.hxx : \
+$(i18npool_CODIR)/lrl_include.hxx : $(SRCDIR)/i18npool/CustomTarget_collator.mk \
 		$(SRCDIR)/i18npool/source/collator/data | $(i18npool_CODIR)/.dir
 	$(call gb_Output_announce,$(subst $(WORKDIR)/,,$@),$(true),ECH,1)
 	echo '#define LOCAL_RULE_LANGS "$(sort $(foreach txt,$(i18npool_COTXTS), \

i18npool/Library_collator_data.mk

@@ -7,6 +7,10 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 
+i18npool_ICULT53 := $(filter 1, $(shell expr $(ICU_MAJOR) \< 53))
+i18npool_LCDALL := $(wildcard $(SRCDIR)/i18npool/source/collator/data/*.txt)
+i18npool_LCDTXTS := $(if $(i18npool_ICULT53), $(i18npool_LCDALL), $(filter-out %/ko_charset.txt, $(i18npool_LCDALL)))
+
 $(eval $(call gb_Library_Library,collator_data))
 
 $(eval $(call gb_Library_use_libraries,collator_data,\
@@ -14,7 +18,7 @@ $(eval $(call gb_Library_use_libraries,collator_data,\
 ))
 
 $(eval $(call gb_Library_add_generated_exception_objects,collator_data,\
-	$(foreach txt,$(wildcard $(SRCDIR)/i18npool/source/collator/data/*.txt),\
+	$(foreach txt,$(i18npool_LCDTXTS),\
 		CustomTarget/i18npool/collator/collator_$(notdir $(basename $(txt)))) \
 ))
 

i18npool/source/collator/collator_unicode.cxx

@@ -182,10 +182,12 @@ Collator_Unicode::loadCollatorAlgorithm(const OUString& rAlgorithm, const lang::
                     func = get_collator_data_ja_phonetic_alphanumeric_last;
 #endif
 #if WITH_LOCALE_ALL || WITH_LOCALE_ko
+#if (U_ICU_VERSION_MAJOR_NUM < 53)
             } else if ( rLocale.Language == "ko" ) {
                 if ( rAlgorithm == "charset" )
                     func = get_collator_data_ko_charset;
 #endif
+#endif
 #if WITH_LOCALE_ALL || WITH_LOCALE_ku
             } else if ( rLocale.Language == "ku" ) {
                 if ( rAlgorithm == "alphanumeric" )