*   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

        }

        /**
         * Sets the maximum size for alphanumeric terms.
         *
         * @param maxAlphanumTermSize the maximum size for alphanumeric terms
         * @return the current instance of {@code TextNormalizeContext} for method chaining
         */
        public TextNormalizeContext maxAlphanumTermSize(final int maxAlphanumTermSize) {
            this.maxAlphanumTermSize = maxAlphanumTermSize;

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

    /**
     * Constructs a new FileTransformer.
     */
    public FileTransformer() {
        // Default constructor
    }

    /**
     * A path to store downloaded files. The default path is a current
     * directory.
     */
    protected String path;

    /**
     * A string to replace ?.
     */
    protected String questionStr = "_QUEST_";

    /**
     * A string to replace :.
     */

        }
        urlFilter.processUrl(url);
    }

    /**
     * Returns the current session ID.
     * @return The session ID.
     */
    public String getSessionId() {
        return crawlerContext.sessionId;
    }

    /**
     * Sets the session ID.
     * If the new session ID is different from the current one, it updates the session ID in the URL queue service.
     * @param sessionId The new session ID.

        } catch (final Exception e) {
            throw new ExtractException(e);
        }
    }

    /**
     * Gets an XPathAPI instance from the cache for the current thread.
     *
     * @return the XPathAPI instance
     */
    protected XPathAPI getXPathAPI() {
        try {
            return xpathAPICache.get(Thread.currentThread().getName());

         */
        public String toUrl() {
            return toUrl(uri.getPath());
        }

        /**
         * Constructs a child URL by appending the specified child path to the current URL.
         *
         * @param child The child path to append
         * @return The complete child URL
         */
        public String toChildUrl(final String child) {
            final String url = toUrl();