/**
     * Output encoding.
     */
    protected String outputEncoding = Constants.UTF_8;

    /**
     * If true, read a content as a text when an extraction fails.
     */
    protected boolean readAsTextIfFailed = false;

    /**
     * Max compression ratio.
     */
    protected long maxCompressionRatio = 100;

    /**

 * </p>
 *
 * <p>
 * The conversion process involves iterating through the input string, identifying Katakana characters,
 * and replacing them with their corresponding Alphabet representations based on the internal mapping.
 * When a Katakana character has multiple Alphabet representations, the converter generates multiple
 * possible readings.
 * </p>
 *
 * <p>

        CrawlerSystemException exception = new CrawlerSystemException(cause);

        assertNotNull(exception);
        assertSame(cause, exception.getCause());
        // When constructed with cause only, message should be cause's toString()
        assertEquals("java.lang.IllegalArgumentException: Root cause", exception.getMessage());
    }

    /**
     * Test constructor with null cause

      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.

   4. Redistribution. You may reproduce and distribute copies of the

    /**
     * Cache for session queues.
     */
    protected Map<String, QueueHolder> sessionCache = new ConcurrentHashMap<>();

    /**
     * The number of URLs to fetch when polling.
     */
    protected int pollingFetchSize = 1000;

    /**
     * The maximum size of the crawling queue.
     */
    protected int maxCrawlingQueueSize = 100;

    /**

      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.

   4. Redistribution. You may reproduce and distribute copies of the

 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

        dataService.delete(sessionId);
        urlFilter.clear();
    }

    /**
     * Closes the crawler, releasing resources.
     * This method is called automatically when the crawler is used in a try-with-resources statement.
     */
    @Override
    public void close() {
        clientFactory.close();
    }

    /**
     * Adds an include filter for URLs.

 * {@link XpathTransformer} is a class that transforms HTML content into XML format based on XPath expressions.
 * It extracts data from an HTML document by applying XPath rules defined in {@link #fieldRuleMap}.
 * The extracted data is then formatted into an XML structure and stored in the {@link ResultData}.
 * <p>
 * This class extends {@link HtmlTransformer} and overrides the {@link #storeData(ResponseData, ResultData)} method

    }

    /**
     * Sets the response body from a file.
     *
     * @param responseBody the file containing the response body
     * @param isTemporary true if the file is temporary and should be deleted when closing
     */
    public void setResponseBody(final File responseBody, final boolean isTemporary) {
        responseBodyFile = responseBody;
        isTemporaryFile = isTemporary;
    }

    /**