- **Flexible Configuration**: XML-based dependency injection with LastaFlute DI
- **Extensible Architecture**: Plugin system for custom extractors, transformers, and clients
- **Rate Limiting**: Politeness policies and interval controllers
- **URL Filtering**: Regex-based inclusion/exclusion patterns
- **Data Persistence**: Multiple backend options including OpenSearch integration

## Technology Stack

<components namespace="fessCrawler">
    <include path="crawler/container.xml"/>
    <include path="crawler/client.xml"/>
    <include path="crawler/rule.xml"/>
    <include path="crawler/filter.xml"/>
    <include path="crawler/interval.xml"/>
    <include path="crawler/extractor.xml"/>
    <include path="crawler/mimetype.xml"/>
    <include path="crawler/encoding.xml"/>
    <include path="crawler/urlconverter.xml"/>
    <include path="crawler/log.xml"/>

        final FaultTolerantClient client = new FaultTolerantClient();
        client.setRetryInterval(100);
        final TestClient testClient = new TestClient();
        testClient.exceptionCount = 4;
        testClient.interval = 100;
        final TestListener testListener = new TestListener();
        client.setCrawlerClient(testClient);
        client.setRequestListener(testListener);
        final String url = "http://test.com/";

    <sub-class-of type="text/plain"/>
    <alias type="text/x-dtd"/>
    <glob pattern="*.dtd"/>
  </mime-type>

  <mime-type type="application/xml-external-parsed-entity">
    <alias type="text/xml-external-parsed-entity"/>
  </mime-type>

  <mime-type type="application/xmpp+xml"/>
  <mime-type type="application/xop+xml">
    <glob pattern="*.xop"/>
  </mime-type>

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval.impl;

import java.util.Map;

import org.codelibs.core.lang.ThreadUtil;

/**
 * Default implementation of the IntervalController.
 * This class provides a default way to manage delays between crawler operations.

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval.impl;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;

     */
    public void setSizeForDelete(final int sizeForDelete) {
        this.sizeForDelete = sizeForDelete;
    }

    /**
     * Sets the retry interval in milliseconds.
     *
     * @param retryInterval The retry interval.
     */
    public void setRetryInterval(final long retryInterval) {
        this.retryInterval = retryInterval;
    }

    /**
     * Sets the maximum retry count.

        } catch (final Exception e) {
            throw new SitemapsException("Could not parse XML Sitemaps.", e);
        }
        return handler.getSitemapSet();
    }

    /**
     * Disables external resources for the SAX parser to prevent XXE attacks.
     * @param parser the SAX parser to configure
     * @throws SAXNotRecognizedException if the parser doesn't recognize the feature

import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.helper.LogHelper;
import org.codelibs.fess.crawler.interval.IntervalController;
import org.codelibs.fess.crawler.log.LogType;
import org.codelibs.fess.crawler.processor.ResponseProcessor;
import org.codelibs.fess.crawler.rule.Rule;
import org.codelibs.fess.crawler.service.DataService;

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the connection check interval in seconds.
     *
     * @param connectionCheckInterval The connection check interval
     */
    public void setConnectionCheckInterval(final int connectionCheckInterval) {
        this.connectionCheckInterval = connectionCheckInterval;
    }