import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.RequestData.Method;

/**
 * Builder class for creating RequestData objects using a fluent interface.
 * This class provides a simple way to construct RequestData objects with method chaining.
 *
 * <p>Usage example:</p>
 * <pre>
 * RequestData request = RequestDataBuilder.newRequestData()
 *     .method("GET")

         */
        void onRequestStart(FaultTolerantClient client, RequestData request);

        /**
         * Called before each request attempt.
         *
         * @param client the fault-tolerant client
         * @param request the request data
         * @param count the current retry count
         */
        void onRequest(FaultTolerantClient client, RequestData request, int count);

        /**

 */
package org.codelibs.fess.crawler.exception;

import java.util.Set;

import org.codelibs.fess.crawler.entity.RequestData;

/**
 * {@link ChildUrlsException} is thrown when child URLs are found during crawling.
 * It extends {@link CrawlerSystemException} and holds a set of {@link RequestData}
 * representing the child URLs that caused the exception.
 *
 */
public class ChildUrlsException extends CrawlerSystemException {

        String requestMethod;

        @Override
        public void onRequestStart(final FaultTolerantClient client, final RequestData request) {
            startCount++;
        }

        @Override
        public void onRequest(final FaultTolerantClient client, final RequestData request, final int count) {
            requestCount++;
            requestUrl = request.getUrl();

            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.
 * It then iterates through the sitemaps in the SitemapSet, extracts the URL
 * from each sitemap, and creates a new {@link RequestData} object for each URL.
 * These RequestData objects are added to a set of child URLs, which are then
 * passed to a {@link ChildUrlsException} to be processed by the crawler.
 * </p>
 *
 * <p>

     * The input object can be either a single string or a list of strings representing URLs.
     *
     * @param obj the object containing anchor URLs (String or List of Strings)
     * @return a set of RequestData objects for the anchor URLs, or null if no valid URLs found
     */
    protected Set<RequestData> getAnchorSet(final Object obj) {

            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(1, urlSet.size());
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                assertTrue(url.contains("dir1/test3.txt"));
            }
        } finally {

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client;

import java.util.Map;

import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * Interface representing a client for a web crawler.
 * This client is responsible for executing requests and handling responses.

     */
    @Override
    protected List<RequestData> convertChildUrlList(final List<RequestData> urlList) {
        if (urlList != null) {
            final PathMappingHelper pathMappingHelper = getPathMappingHelper();
            for (final RequestData requestData : urlList) {
                String url = requestData.getUrl();