return fessConfig.isCrawlerDocumentDuplicateTermRemoved();
    }

    protected int[] getSpaceChars() {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        return fessConfig.getCrawlerDocumentSpaceCharsAsArray();
    }

    public String getDigest(final ResponseData responseData, final String content, final Map<String, Object> dataMap, final int maxWidth) {
        if (content == null) {

            @Override
            public String getCrawlerDocumentSpaceChars() {
                return "u0020u3000";
            }
        };

        int[] chars = fessConfig.getCrawlerDocumentSpaceCharsAsArray();
        assertEquals(2, chars.length);
        assertEquals(32, chars[0]);
        assertEquals(12288, chars[1]);
    }

    public void test_getCrawlerDocumentFullstopCharsAsArray() {

        return requiredValidator.isValid(value, null);
    }

    String getCrawlerDocumentSpaceChars();

    default int[] getCrawlerDocumentSpaceCharsAsArray() {
        return getCrawlerDocumentCharsAsArray(CRAWLER_DOCUMENT_SPACE_CHARS, getCrawlerDocumentSpaceChars());
    }

    default String[] getCrawlerDocumentSpaces() {