return fessConfig.isCrawlerDocumentDuplicateTermRemoved();
    }

    /**
     * Gets the array of space character codes from configuration.
     *
     * @return array of character codes to be treated as spaces
     */
    protected int[] getSpaceChars() {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();

        assertEquals("line1 line2 line3", activityHelper.normalizeScript("line1\nline2\rline3"));
        assertEquals("tab1_tab2", activityHelper.normalizeScript("tab1\ttab2"));
        // \t -> _, \n -> space, \r -> space, \n -> space
        assertEquals("mixed_   ", activityHelper.normalizeScript("mixed\t\n\r\n"));
    }

    @Test
    public void test_normalizeScript_longScript() {

    default String[] getCrawlerDocumentSpaces() {
        String[] spaces = (String[]) propMap.get(CRAWLER_DOCUMENT_SPACES);
        if (spaces == null) {
            spaces = Arrays.stream(getCrawlerDocumentSpaceCharsAsArray()).mapToObj(Character::toString).toArray(n -> new String[n]);
            propMap.put(CRAWLER_DOCUMENT_SPACES, spaces);
        }
        return spaces;
    }

        assertEquals("test{}|\\", result);

        result = systemHelper.encodeUrlFilter("test^path");
        assertEquals("test^path", result);

        result = systemHelper.encodeUrlFilter("test space");
        assertEquals("test+space", result);
    }

    @Test
    public void test_normalizeLang_caseVariations() {
        assertEquals("ja", systemHelper.normalizeLang("JA"));

# Maximum size of symbol terms in documents.
crawler.document.max.symbol.term.size=10
# Whether to remove duplicate terms in documents.
crawler.document.duplicate.term.removed=false
# Unicode space characters for document parsing.
crawler.document.space.chars=u0009u000Au000Bu000Cu000Du001Cu001Du001Eu001Fu0020u00A0u1680u180Eu2000u2001u2002u2003u2004u2005u2006u2007u2008u2009u200Au200Bu200Cu202Fu205Fu3000uFEFFuFFFDu00B6

    /**
     * Get the value for the key 'crawler.document.space.chars'. <br>
     * The value is, e.g. u0009u000Au000Bu000Cu000Du001Cu001Du001Eu001Fu0020u00A0u1680u180Eu2000u2001u2002u2003u2004u2005u2006u2007u2008u2009u200Au200Bu200Cu202Fu205Fu3000uFEFFuFFFDu00B6 <br>
     * comment: Unicode space characters for document parsing.