return fessConfig.isCrawlerDocumentDuplicateTermRemoved();
    }

    /**
     * Gets the array of space character codes from configuration.
     *
     * @return array of character codes to be treated as spaces
     */
    protected int[] getSpaceChars() {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();

        assertEquals("line1 line2 line3", activityHelper.normalizeScript("line1\nline2\rline3"));
        assertEquals("tab1_tab2", activityHelper.normalizeScript("tab1\ttab2"));
        // \t -> _, \n -> space, \r -> space, \n -> space
        assertEquals("mixed_   ", activityHelper.normalizeScript("mixed\t\n\r\n"));
    }

    @Test
    public void test_normalizeScript_longScript() {

        assertEquals("test{}|\\", result);

        result = systemHelper.encodeUrlFilter("test^path");
        assertEquals("test^path", result);

        result = systemHelper.encodeUrlFilter("test space");
        assertEquals("test+space", result);
    }

    @Test
    public void test_normalizeLang_caseVariations() {
        assertEquals("ja", systemHelper.normalizeLang("JA"));

# Maximum size of symbol terms in documents.
crawler.document.max.symbol.term.size=10
# Whether to remove duplicate terms in documents.
crawler.document.duplicate.term.removed=false
# Unicode space characters for document parsing.
crawler.document.space.chars=u0009u000Au000Bu000Cu000Du001Cu001Du001Eu001Fu0020u00A0u1680u180Eu2000u2001u2002u2003u2004u2005u2006u2007u2008u2009u200Au200Bu200Cu202Fu205Fu3000uFEFFuFFFDu00B6