/**
     * Output encoding.
     */
    protected String outputEncoding = Constants.UTF_8;

    /**
     * If true, read a content as a text when an extraction fails.
     */
    protected boolean readAsTextIfFailed = false;

    /**
     * Max compression ratio.
     */
    protected long maxCompressionRatio = 100;

    /**
     * Max uncompression size.
     */

     * @return true if the stream contains valid sitemap data, false otherwise
     */
    protected boolean isValid(final InputStream in, final boolean recursive) {
        final BufferedInputStream bis = new BufferedInputStream(in);
        bis.mark(preloadSize);

        final byte[] bytes = new byte[preloadSize];
        try {
            if (bis.read(bytes) == -1) {
                return false;

     * @return true if SIDs should be resolved, false otherwise
     */
    public boolean isResolveSids() {
        return resolveSids;
    }

    /**
     * Sets whether SIDs (Security Identifiers) should be resolved.
     *
     * @param resolveSids true to resolve SIDs, false otherwise
     */
    public void setResolveSids(final boolean resolveSids) {
        this.resolveSids = resolveSids;
    }

    /**

     */
    protected boolean namespaceAware;

    /**
     * If true, the parser will convert CDATA nodes to Text nodes and append them to the adjacent Text node.
     */
    protected boolean coalescing;

    /**
     * If true, the parser will expand entity reference nodes.
     */
    protected boolean expandEntityRef = false;

    /**

    private final List<String> languages = new ArrayList<>();

    private boolean suggestDetail = true;

    private ReadingConverter readingConverter;

    private Normalizer normalizer;

    private float prefixMatchWeight = 2.0f;

    private boolean matchWordFirst = true;

    private boolean skipDuplicateWords = true;

    /**
     * Sets the index name.

    public static final String DEFAULT_DELIMITERS = " 　\t\n\r";

    static final int BUFFER_SIZE = 4096;

    private final int n;

    private final String delimiters;

    private final boolean expand;

    private final boolean ignoreCase;

    private final SynonymLoader synonymLoader;

    private long lastModified;

    private SynonymMap synonymMap = null;

    private FST.Arc<BytesRef> scratchArc;

    private static class TestReadingConverter implements ReadingConverter {

        private boolean initialized = false;
        private int initCount = 0;
        private boolean japaneseMode = false;

        public void setJapaneseMode(boolean japaneseMode) {
            this.japaneseMode = japaneseMode;
        }

        public boolean isInitialized() {
            return initialized;
        }

        // Access protected constructor via reflection
        Constructor<CrawlerSystemException> constructor =
                CrawlerSystemException.class.getDeclaredConstructor(String.class, boolean.class, boolean.class);
        constructor.setAccessible(true);

        // Test with suppression enabled and writable stack trace
        String message1 = "Test with suppression enabled";

     */
    protected boolean passiveNatWorkaround;

    /**
     * The report active external host.
     */
    protected String reportActiveExternalHost;

    /**
     * Whether to use EPSV with IPv4.
     */
    protected boolean useEPSVwithIPv4;

    /**
     * Whether to use implicit SSL/TLS encryption.
     */
    protected Boolean isImplicit;

    /**

        return new TextNormalizeContext(reader);
    }

    private static boolean isLastSpaceChar(final StringBuilder buf) {
        if (buf.length() == 0) {
            return false;
        }
        return buf.charAt(buf.length() - 1) == ' ';
    }

    private static boolean removeLastDuplication(final StringBuilder buf, final int size, final boolean isSpace,
            final Set<String> termCache) {