/** The URL for this request. */
    private String url;

    /** The weight/priority of this request (default: 1.0). */
    private float weight = 1.0f;

    /**
     * Creates a new RequestData instance.
     */
    public RequestData() {
        super();
    }

    /**
     * Gets the HTTP method for this request.
     * @return the HTTP method
     */
    public Method getMethod() {
        return method;

     * @param url the request data to add to the child URL set
     */
    public void addUrl(final RequestData url) {
        childUrlSet.add(url);
    }

    /**
     * Add child URLs.
     * @param c the collection of request data to add to the child URL set
     */
    public void addAllUrl(final Collection<RequestData> c) {
        if (c != null) {
            childUrlSet.addAll(c);
        }
    }

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.RequestData.Method;

/**
 * Builder class for creating RequestData objects using a fluent interface.
 * This class provides a simple way to construct RequestData objects with method chaining.
 *
 * <p>Usage example:</p>
 * <pre>
 * RequestData request = RequestDataBuilder.newRequestData()
 *     .method("GET")

         */
        void onRequestStart(FaultTolerantClient client, RequestData request);

        /**
         * Called before each request attempt.
         *
         * @param client the fault-tolerant client
         * @param request the request data
         * @param count the current retry count
         */
        void onRequest(FaultTolerantClient client, RequestData request, int count);

        /**

 */
package org.codelibs.fess.crawler.exception;

import java.util.Set;

import org.codelibs.fess.crawler.entity.RequestData;

/**
 * {@link ChildUrlsException} is thrown when child URLs are found during crawling.
 * It extends {@link CrawlerSystemException} and holds a set of {@link RequestData}
 * representing the child URLs that caused the exception.
 *
 */
public class ChildUrlsException extends CrawlerSystemException {

            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.
 * It then iterates through the sitemaps in the SitemapSet, extracts the URL
 * from each sitemap, and creates a new {@link RequestData} object for each URL.
 * These RequestData objects are added to a set of child URLs, which are then
 * passed to a {@link ChildUrlsException} to be processed by the crawler.
 * </p>
 *
 * <p>

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client;

import java.util.Map;

import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * Interface representing a client for a web crawler.
 * This client is responsible for executing requests and handling responses.

import org.codelibs.fess.crawler.client.smb.SmbClient;
import org.codelibs.fess.crawler.client.storage.StorageClient;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;
import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */

import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.MaxLengthExceededException;