* This class provides methods for managing a queue of URLs to be crawled,
 * including adding, deleting, and retrieving URLs from the queue.
 * It uses a {@link MemoryDataHelper} to store the URL queue data in memory.
 *
 * <p>
 * The class is responsible for:
 * </p>
 * <ul>
 *   <li>Updating session IDs for URL queues.</li>
 *   <li>Adding new URLs to the queue.</li>

 * <ul>
 *   <li>Before processing a URL ({@link #delayBeforeProcessing()})</li>
 *   <li>After processing a URL ({@link #delayAfterProcessing()})</li>
 *   <li>When there are no URLs in the queue ({@link #delayAtNoUrlInQueue()})</li>
 *   <li>While waiting for new URLs to be added to the queue ({@link #delayForWaitingNewUrl()})</li>
 * </ul>
 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay

    protected String transformerName;

    /** The data of the result. */
    protected byte[] data;

    /** The encoding of the result. */
    protected String encoding;

    /** The set of the child URLs. */
    protected Set<RequestData> childUrlSet = new LinkedHashSet<>();

    /** The raw data. */
    protected Object rawData = null;

    /** The serializer. */
    protected Function<Object, byte[]> serializer;

    // public ResponseData call() throws Exception {
    // String[] urls =
    // new String[] {
    // "http://.../",
    // "http://.../test.pdf",
    // "http://.../test.doc",
    // "http://.../test.xls",
    // "http://.../test.ppt",
    // "http://.../test.txt", };
    // for (String url : urls) {
    // ResponseData responseData = httpClient.doGet(url);

import org.xml.sax.InputSource;

/**
 * {@link XpathTransformer} is a class that transforms HTML content into XML format based on XPath expressions.
 * It extracts data from an HTML document by applying XPath rules defined in {@link #fieldRuleMap}.
 * The extracted data is then formatted into an XML structure and stored in the {@link ResultData}.
 * <p>

     * Test match with no patterns configured
     */
    public void test_match_noPatterns() {
        String sessionId = "test-session-009";
        urlFilter.init(sessionId);

        // Without any patterns, all URLs should match
        assertTrue(urlFilter.match("https://example.com/"));
        assertTrue(urlFilter.match("https://test.com/page"));
        assertTrue(urlFilter.match("ftp://files.com/document.pdf"));

            logHelper.log(key, objs);
        }
    }

    /**
     * Runs the crawling process in a separate thread.
     * This method fetches URLs from the queue, accesses content, processes responses,
     * and extracts child URLs until the crawling process is done or no more URLs are available.
     */
    @Override
    public void run() {
        log(logHelper, LogType.START_THREAD, crawlerContext);

 * <p>
 * The class contains methods for logging different types of events, such as:
 * </p>
 * <ul>
 *   <li>Starting and finishing threads</li>
 *   <li>Starting and cleaning up crawling</li>
 *   <li>Handling unsupported URLs</li>
 *   <li>Checking last modified dates</li>
 *   <li>Getting content</li>
 *   <li>Handling redirects</li>
 *   <li>Processing responses</li>
 *   <li>Handling exceptions during crawling and child URL processing</li>

 *
 * <p>
 * It provides methods to set and retrieve the server address, port, username,
 * password, and domain. Additionally, it offers a method to construct a path
 * prefix for SMB1 URLs based on the configured server and port.
 * </p>
 *
 * <p>
 * The path prefix is in the format "smb1://server:port/", where the port is
 * included only if it's greater than 0. If the server is not set, the path

                logger.debug("CrawlerClientCreator is unavailable.", e);
            }
        }
    }

    /**
     * Adds a client with a regular expression pattern.
     * @param regex The regular expression to match URLs.
     * @param client The CrawlerClient instance.
     */
    public void addClient(final String regex, final CrawlerClient client) {
        if (StringUtil.isBlank(regex)) {