* represents a valid sitemap. It uses a SitemapsHelper to validate the response body as an InputStream.
 * The rule checks if the URL matches the defined regex pattern and then validates the content as a sitemap.
 * If any exception occurs during the sitemap validation, it logs the error and returns false.
 *
 */
public class SitemapsRule extends RegexRule {
    /**

 * and provides getter and setter methods for common properties such as
 * {@code ruleId} and {@code responseProcessor}.
 * </p>
 *
 * <p>
 * Subclasses should extend this class and implement the abstract methods
 * defined in the {@link Rule} interface to provide specific rule logic.
 * </p>
 *
 */
public abstract class AbstractRule implements Rule {

    private static final long serialVersionUID = 1L;

    /** The rule ID. */

        }
    }

    @Test
    public void testAnalyzeWithPunctuation() {
        // Test analyze with various punctuation
        String text = "Hello, world! How are you? I'm fine.";
        String field = "content";
        String lang = "en";

        List<AnalyzeToken> tokens = analyzer.analyze(text, field, lang);

        assertNotNull(tokens);

 *   <li>Loading index settings and mappings from resources.</li>
 *   <li>Checking the availability of analyzers for supported languages.</li>
 * </ul>
 *
 * <p>Supported languages are defined in the SUPPORTED_LANGUAGES array.</p>
 *
 * <p>Inner class:</p>
 * <ul>
 *   <li>DefaultContentsAnalyzer: Implements the SuggestAnalyzer interface to analyze text and retrieve tokens using

 *   <li>While waiting for new URLs to be added to the queue ({@link #delayForWaitingNewUrl()})</li>
 * </ul>
 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay
 * mechanism for each of these stages.
 * </p>
 *
 * <p>
 * The class also provides a mechanism to ignore exceptions that may occur during the delay process.

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.processor;

import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * The ResponseProcessor interface defines a contract for processing response data.
 * Implementations of this interface are responsible for handling the response data
 * obtained during a crawling process.
 */
public interface ResponseProcessor {

    /**

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval;

/**
 * The {@code IntervalController} interface defines methods for controlling
 * the delay intervals in a web crawler. It includes constants representing
 * different types of processing states and a method to introduce a delay
 * based on the type of processing.
 * <p>

 * The class supports caching of include and exclude patterns for scenarios where a session ID is not available.
 * It also provides methods to initialize the filter with a session ID, clear the filter,
 * match a URL against the defined patterns, and process a URL to add include or exclude patterns based on predefined filtering patterns.
 *
 */
/**
 * This class is an implementation of a URL filter.
 */
public class UrlFilterImpl implements UrlFilter {

package org.codelibs.fess.crawler.rule;

import java.io.Serializable;

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

/**
 * The Rule interface defines the contract for implementing rules that can be applied to
 * response data in a web crawler. Implementations of this interface should provide logic

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.log;

/**
 * LogType defines the different types of log messages that can be generated during the crawling process.
 * Each enum constant represents a specific event or state in the crawler's execution.
 */
public enum LogType {