* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.processor.impl;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;

import org.codelibs.fess.crawler.processor.ResponseProcessor;

/**
 * The Rule interface defines the contract for implementing rules that can be applied to
 * response data in a web crawler. Implementations of this interface should provide logic
 * to determine if a given response data matches the rule, retrieve the rule's identifier,
 * and obtain the associated response processor.
 */
public interface Rule extends Serializable {

        connect(con -> {
            final RequestProcessor processor = new RequestProcessor(encoding, threshold);
            processor.accept(con);
            try (final CurlResponse res = processor.getResponse()) {
                actionListener.accept(res);
            } catch (final IOException e) {
                exceptionListener.accept(e);

     * @return The query class name.
     */
    protected abstract String getQueryClassName();

    /**
     * Registers this query command with the query processor.
     * Associates this command with its query class name in the processor.
     */
    public void register() {
        ComponentUtil.getQueryProcessor().add(getQueryClassName(), this);
    }

    /**
     * Gets the query field configuration.

		<property name="ruleId">"sitemapsRule"</property>
		<property name="responseProcessor">
			<component
				class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>".*sitemap.*"</arg>
		</postConstruct>
	</component>

import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.helper.SitemapsHelper;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

import jakarta.annotation.Resource;

/**
 * A response processor implementation that handles sitemaps.
 * It parses the response body as a SitemapSet, extracts URLs from the sitemaps,
 * and adds them as child URLs to be crawled.
 *
 * <p>

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.processor.impl;

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

/**
 * NullResponseProcessor is a class that implements the ResponseProcessor interface.

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.processor;

import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * The ResponseProcessor interface defines a contract for processing response data.

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.processor.impl;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import org.apache.logging.log4j.LogManager;

/**
 * RankFusionProcessor manages multiple search engines and combines their results using rank fusion algorithms.
 * This processor supports searching with multiple searchers concurrently and merging their results based on
 * ranking scores to provide more comprehensive and accurate search results.
 *
 * The processor maintains a pool of searchers and an executor service for concurrent operations.