throw new CrawlerSystemException("CrawlerClient is null.");
        }
        int current = 0;
        boolean added = false;
        final Map<Pattern, CrawlerClient> newClientMap = new LinkedHashMap<>();
        for (final Map.Entry<Pattern, CrawlerClient> entry : clientMap.entrySet()) {
            if (pos == current) {
                newClientMap.put(Pattern.compile(regex), client);
                added = true;

 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

            final File tempDir = File.createTempFile("crawlerDocRoot", "");
            tempDir.delete();
            tempDir.mkdirs();

            final StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

            tempDir.delete();
            tempDir.mkdirs();

            // robots.txt
            StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

    /**
     * Constructs a new FileTransformer.
     */
    public FileTransformer() {
        // Default constructor
    }

    /**
     * A path to store downloaded files. The default path is a current
     * directory.
     */
    protected String path;

    /**
     * A string to replace ?.
     */
    protected String questionStr = "_QUEST_";

    /**
     * A string to replace :.
     */

        /**
         * Called before each request attempt.
         *
         * @param client the fault-tolerant client
         * @param request the request data
         * @param count the current retry count
         */
        void onRequest(FaultTolerantClient client, RequestData request, int count);

        /**
         * Called when a request ends.
         *

    /**
     * Sets the encoding for the URL queue.
     *
     * @param encoding the encoding to be set
     */
    void setEncoding(String encoding);

    /**
     * Retrieves the parent URL of the current URL in the queue.
     *
     * @return the parent URL as a String.
     */
    String getParentUrl();

    /**
     * Sets the parent URL for this URL queue entry.
     *

        }
        urlFilter.processUrl(url);
    }

    /**
     * Returns the current session ID.
     * @return The session ID.
     */
    public String getSessionId() {
        return crawlerContext.sessionId;
    }

    /**
     * Sets the session ID.
     * If the new session ID is different from the current one, it updates the session ID in the URL queue service.
     * @param sessionId The new session ID.

    /** The metadata associated with the URL queue. */
    protected String metaData;

    /** The encoding of the URL queue. */
    protected String encoding;

    /** The parent URL of the current URL in the queue. */
    protected String parentUrl;

    /** The depth of the URL in the queue. */
    protected Integer depth;

    /** The last modified timestamp of the URL in the queue. */