public int nextToken() {
        initVal();
        if (processEOF()) {
            return ttype;
        }
        if (processWhitespace()) {
            return ttype;
        }
        if (processWord()) {
            return ttype;
        }
        if (processQuote()) {
            return ttype;
        }
        if (processOrdinary()) {
            return ttype;
        }
        ttype = peekc;

 */
package org.codelibs.fess.crawler.processor;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;
import org.codelibs.fess.ingest.IngestFactory;

	<component name="sitemapsRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"sitemapsRule"</property>
		<property name="responseProcessor">
			<component class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>

    }

    @Override
    public ResponseProcessor getResponseProcessor() {
        return responseProcessor;
    }

    /**
     * Sets the response processor for this rule.
     *
     * @param responseProcessor the response processor to set
     */
    public void setResponseProcessor(final ResponseProcessor responseProcessor) {
        this.responseProcessor = responseProcessor;
    }

        // Test constructor
        NullResponseProcessor processor = new NullResponseProcessor();
        assertNotNull(processor);
    }

    public void test_implementsResponseProcessor() {
        // Test that NullResponseProcessor implements ResponseProcessor
        NullResponseProcessor processor = new NullResponseProcessor();
        assertTrue(processor instanceof ResponseProcessor);
    }

        processor.setNotModifiedHttpCodes(new int[] { 304 });

        ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        assertTrue(processor.isSuccessful(responseData));

        responseData.setHttpStatusCode(100);
        assertFalse(processor.isSuccessful(responseData));
        responseData.setHttpStatusCode(304);
        assertFalse(processor.isSuccessful(responseData));

import org.opensearch.index.query.QueryBuilder;

import jakarta.annotation.PostConstruct;

/**
 * Query processor component that handles query filters and commands.
 * This class provides a pipeline for processing Lucene queries by applying
 * a chain of filters and executing registered query commands.
 *
 * <p>The processor maintains a map of query commands indexed by query class names

import org.codelibs.fess.crawler.processor.ResponseProcessor;

/**
 * The Rule interface defines the contract for implementing rules that can be applied to
 * response data in a web crawler. Implementations of this interface should provide logic
 * to determine if a given response data matches the rule, retrieve the rule's identifier,
 * and obtain the associated response processor.
 */
public interface Rule extends Serializable {

		<property name="ruleId">"sitemapsRule"</property>
		<property name="responseProcessor">
			<component
				class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>".*sitemap.*"</arg>
		</postConstruct>
	</component>

import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.helper.SitemapsHelper;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

import jakarta.annotation.Resource;

/**
 * A response processor implementation that handles sitemaps.
 * It parses the response body as a SitemapSet, extracts URLs from the sitemaps,
 * and adds them as child URLs to be crawled.
 *
 * <p>