fix ignoreCharsets param of Icu4jEncodingDetector
MODIFY
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
@@ -75,7 +75,7 @@ public class Icu4jEncodingDetector implements EncodingDetector {
try {
String n = match.getNormalizedName();
if (ignoreCharsets.contains(n)) {
- return null;
+ continue;
}
return CharsetUtils.forName(match.getNormalizedName());
} catch (IllegalArgumentException e) {

MODIFY
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java
@@ -136,10 +136,16 @@ public class CharsetDetectorTest extends TikaTest {
TikaConfig tikaConfig = new TikaConfig(
getResourceAsStream("/test-configs/tika-config-ignore-charset.xml"));
- Metadata m = new Metadata();
+ AutoDetectParser parser = new AutoDetectParser(tikaConfig);
+ Metadata m = new Metadata();
m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
assertContains("ACTIVE AGE", getXML("testIgnoreCharset.txt",
- new AutoDetectParser(tikaConfig), m).xml);
+ parser, m).xml);
+
+ m = new Metadata();
+ m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
+ assertContains("Please check your email", getXML("test_ignore_IBM420.html",
+ parser, m).xml);
}
}

MODIFY
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-configs/tika-config-ignore-charset.xml
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-configs/tika-config-ignore-charset.xml
<encodingDetector class="org.apache.tika.parser.txt.Icu4jEncodingDetector">
<params>
<param name="ignoreCharsets" type="list">
+ <string>IBM420</string>
<string>IBM424</string>
</param>
</params>

ADD
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/test_ignore_IBM420.html
tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/test_ignore_IBM420.html
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/resources/test-documents/test_ignore_IBM420.html differ

TIKA-3774: fix ignoreCharsets param of Icu4jEncodingDetector