}

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

# Lasta Di properties, you can set container's options
# _/_/_/_/_/_/_/_/_/_/

# location of smart-deploy mode e.g. maihama_env.properties: lasta_di.smart.deploy.mode
#smart.deploy.mode.location = maihama_env.properties: lasta_di.smart.deploy.mode

# package for smart deploy target e.g. org.docksidestage.app
smart.package1 = org.codelibs.fess.crawler

        return stripInvalidXMLCharacters(//
                value//
                        .replace("&", "&") //
                        .replace("<", "&lt;")//
                        .replace(">", "&gt;")//
                        .replace("\"", "&quot;")//
                        .replace("\'", "&apos;")//
        );
    }

    /**
     * Strips invalid XML characters from the input string.

a?h[i.call(a)]||"object":typeof a},globalEval:function(a){var b,c=eval;a=n.trim(a),a&&(1===a.indexOf("use strict")?(b=l.createElement("script"),b.text=a,l.head.appendChild(b).parentNode.removeChild(b)):c(a))},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=s(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else...

    int NO_URL_IN_QUEUE = 4;

    /** Constant indicating that the crawler is waiting for new URLs. */
    int WAIT_NEW_URL = 8;

    /**
     * Introduces a delay based on the specified type.
     *
     * @param type the type of delay to be introduced
     */
    void delay(int type);

    /** Delay in milliseconds after processing a URL */
    protected long delayMillisAfterProcessing = 0L;

    /** Delay in milliseconds when no URL is in the queue */
    protected long delayMillisAtNoUrlInQueue = 500L;

    /** Delay in milliseconds before processing a URL */
    protected long delayMillisBeforeProcessing = 0L;

    /** Delay in milliseconds for waiting for new URLs */

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay
 * mechanism for each of these stages.
 * </p>
 *
 * <p>
 * The class also provides a mechanism to ignore exceptions that may occur during the delay process.
 * If {@link #ignoreException} is set to true, any exceptions thrown during the delay will be caught
 * and ignored. Otherwise, they will be re-thrown as {@link CrawlerSystemException}.

     */
    protected String path;

    /**
     * A string to replace ?.
     */
    protected String questionStr = "_QUEST_";

    /**
     * A string to replace :.
     */
    protected String colonStr = "_CLN_";

    /**
     * A string to replace ;.
     */
    protected String semicolonStr = "_SCLN_";

    /**
     * A string to replace &.
     */
    protected String ampersandStr = "_AMP_";

 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.
 */