- **Flexible Configuration**: XML-based dependency injection with LastaFlute DI
- **Extensible Architecture**: Plugin system for custom extractors, transformers, and clients
- **Rate Limiting**: Politeness policies and interval controllers
- **URL Filtering**: Regex-based inclusion/exclusion patterns
- **Data Persistence**: Multiple backend options including OpenSearch integration

## Technology Stack

<components namespace="fessCrawler">
    <include path="crawler/container.xml"/>
    <include path="crawler/client.xml"/>
    <include path="crawler/rule.xml"/>
    <include path="crawler/filter.xml"/>
    <include path="crawler/interval.xml"/>
    <include path="crawler/extractor.xml"/>
    <include path="crawler/mimetype.xml"/>
    <include path="crawler/encoding.xml"/>
    <include path="crawler/urlconverter.xml"/>
    <include path="crawler/log.xml"/>

        final FaultTolerantClient client = new FaultTolerantClient();
        client.setRetryInterval(100);
        final TestClient testClient = new TestClient();
        testClient.exceptionCount = 4;
        testClient.interval = 100;
        final TestListener testListener = new TestListener();
        client.setCrawlerClient(testClient);
        client.setRequestListener(testListener);
        final String url = "http://test.com/";

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval.impl;

import java.util.Map;

import org.codelibs.core.lang.ThreadUtil;

/**
 * Default implementation of the IntervalController.
 * This class provides a default way to manage delays between crawler operations.

                            buf.append('\n');
                        } catch (final Exception e) {
                            if (logger.isDebugEnabled()) {
                                logger.debug("Exception in an internal extractor.", e);
                            }
                        } finally {
                            CloseableUtil.closeQuietly(is);
                        }
                    }
                }

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval.impl;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;

 * methods to add, remove, clear, and retrieve data from these structures.
 *
 * <p>The class is thread-safe due to the use of {@code synchronized} keyword on methods that
 * modify the internal data structures. The internal maps are also declared as {@code volatile}
 * to ensure visibility of changes across threads.
 *
 * <p>The class also provides methods to manage URL patterns for inclusion and exclusion, which

 * </p>
 *
 * <p>
 * The conversion process involves iterating through the input string, identifying Katakana characters,
 * and replacing them with their corresponding Alphabet representations based on the internal mapping.
 * When a Katakana character has multiple Alphabet representations, the converter generates multiple
 * possible readings.
 * </p>
 *
 * <p>

     */
    public void setSizeForDelete(final int sizeForDelete) {
        this.sizeForDelete = sizeForDelete;
    }

    /**
     * Sets the retry interval in milliseconds.
     *
     * @param retryInterval The retry interval.
     */
    public void setRetryInterval(final long retryInterval) {
        this.retryInterval = retryInterval;
    }

    /**
     * Sets the maximum retry count.

         * @throws IllegalArgumentException If the arguments are invalid
         * @throws InsufficientDataException If insufficient data is available
         * @throws InternalException If an internal error occurs
         * @throws InvalidResponseException If the response is invalid
         * @throws NoSuchAlgorithmException If the algorithm is not available
         * @throws XmlParserException If XML parsing fails