private final List<String> sitemapList = new ArrayList<>();

    /**
     * Creates a new RobotsTxt instance.
     */
    public RobotsTxt() {
        // Default constructor
    }

    /**
     * Checks if access to a given path is allowed for a specific user agent according to robots.txt rules.
     *
     * @param path The path to check for access permission

 * It also integrates with other Fess Crawler components, such as {@link ContentLengthHelper} and
 * {@link MimeTypeHelper}, to handle content length checks and MIME type detection.
 * </p>
 *
 * <p>
 * The class uses JCIFS properties to configure the SMB connection.
 * </p>
 *
 * <p>
 * Usage example:
 * </p>
 *
 * <pre>
 * {@code

     */
    protected void finishCrawling() {
        synchronized (crawlerContext.activeThreadCountLock) {
            crawlerContext.activeThreadCount--;
        }
    }

    /**
     * Checks if the crawling process should continue.
     * @param tcCount The thread check count.
     * @return true if the crawling should continue, false otherwise.
     */
    protected boolean isContinue(final int tcCount) {

     *
     * @param statusCode the HTTP status code to set
     */
    public void setHttpStatusCode(final int statusCode) {
        httpStatusCode = statusCode;
    }

    /**
     * Checks if this response has a response body.
     *
     * @return true if the response has a body (either as bytes or file), false otherwise
     */
    public boolean hasResponseBody() {

     * Returns the crawler client factory.
     * @return The CrawlerClientFactory instance.
     */
    public CrawlerClientFactory getClientFactory() {
        return clientFactory;
    }

    /**
     * Checks if the crawler is running in background mode.
     * @return true if in background mode, false otherwise.
     */
    public boolean isBackground() {
        return background;
    }

    /**

     *
     * @param sessionId The session ID.
     */
    @Override
    public void saveSession(final String sessionId) {
        // TODO use cache
    }

    /**
     * Checks if a URL has been visited by looking in both the queue and access results.
     *
     * @param urlQueue The URL queue entry to check.
     * @return true if the URL has been visited, false otherwise.
     */

        final int commentIndex = line.indexOf('#');
        if (commentIndex != -1) {
            return line.substring(0, commentIndex);
        }
        return line;
    }

    /**
     * Checks if robots.txt processing is enabled.
     * @return true if enabled, false otherwise
     */
    public boolean isEnabled() {
        return enabled;
    }

    /**

     *
     * @param activeMaxPort The maximum port number
     */
    public void setActiveMaxPort(final int activeMaxPort) {
        this.activeMaxPort = activeMaxPort;
    }

    /**
     * Checks if automatic UTF-8 encoding detection is enabled.
     *
     * @return True if auto-detection is enabled, false otherwise
     */
    public boolean isAutodetectEncoding() {
        return autodetectEncoding;
    }

            }
            resultData.addUrl(RequestDataBuilder.newRequestData().get().url(redirectUrl).build());
        }

        return resultData;
    }

    /**
     * Checks if the response data represents HTML content.
     *
     * @param responseData the response data to check
     * @return true if the content is HTML, false otherwise
     */

                getInitParameter("writeTimeout", (long) 10000, Long.class), getInitParameter("readTimeout", (long) 10000, Long.class));

        isInit = true;
    }

    /**
     * Checks if a bucket exists in the object storage.
     * @param name the name of the bucket to check
     * @return true if the bucket exists, false otherwise