httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

 * during the crawling process. It extracts data, identifies child URLs, and handles
 * character set encoding.
 * <p>
 * This class extends {@link AbstractTransformer} and utilizes various helper classes
 * and components from the Fess Crawler framework, such as {@link CrawlerContainer},
 * {@link UrlConvertHelper}, and {@link EncodingHelper}.
 * </p>
 * <p>
 * The transformation process involves:
 * </p>
 * <ol>

        // Initial state
        assertNull(testRule.getResponseProcessor());

        // Set response processor
        TestResponseProcessor processor = new TestResponseProcessor();
        testRule.setResponseProcessor(processor);
        assertSame(processor, testRule.getResponseProcessor());

        // Change response processor
        TestResponseProcessor newProcessor = new TestResponseProcessor();

    /**
     * Test processUrl method with various URL patterns
     */
    public void test_processUrl_basic() {
        String sessionId = "test-session-012";
        urlFilter.init(sessionId);

        // Process URL should handle different URL formats
        urlFilter.processUrl("https://example.com/");
        urlFilter.processUrl("http://test.com/path/to/page");
        urlFilter.processUrl("ftp://files.server.com/documents/");

     */
    @Override
    public ResponseData doGet(final String uri) {
        return processRequest(uri, true);
    }

    /**
     * Processes a request for the given URI.
     *
     * @param uri the URI to process
     * @param includeContent whether to include content in the response
     * @return the response data
     * @throws CrawlingAccessException if the request fails
     */

        } catch (final Exception e) {
            throw new CrawlingAccessException("Could not access bucket:" + name, e);
        }
    }

    /**
     * Processes a storage request with timeout management.
     * @param uri the URI to process
     * @param includeContent whether to include the actual content in the response
     * @return the response data for the request

import org.codelibs.core.stream.StreamUtil;
import org.codelibs.fess.crawler.Constants;

/**
 * The FormScheme class implements the AuthScheme interface to provide
 * form-based authentication for HTTP clients. It handles the process of
 * obtaining a token and logging in using the provided credentials.
 *
 * <p>This class supports both GET and POST methods for token and login
 * requests. It also allows for the replacement of placeholders in URLs and

 * of the document. It also handles zip bomb prevention and embedded document extraction.
 * </p>
 *
 * <p>
 * The {@link ContentWriter} functional interface is used to abstract the process of writing content to a writer.
 * </p>
 *
 * <p>
 * The class uses temporary files for processing large input streams and ensures that these files are deleted after
 * processing.
 * </p>
 *
 */

        }
    }

    /**
     * Indexes documents from a query log reader asynchronously.
     * @param queryLogReader The query log reader.
     * @param docPerReq The number of documents to process per request.
     * @param requestInterval The interval between requests.
     * @return A Promise that will be resolved with the SuggestIndexResponse or rejected with an error.
     */

    }

    /**
     * Test stateful transformer
     */
    public void test_statefulTransformer() {
        StatefulTransformer transformer = new StatefulTransformer("statefulTransformer");

        // Process multiple URLs
        String[] urls = { "http://example1.com", "http://example2.com", "http://example3.com" };

        for (String url : urls) {
            ResponseData responseData = new ResponseData();