<artifactId>maven-surefire-plugin</artifactId>
			</plugin>
			<plugin>
				<groupId>org.jacoco</groupId>
				<artifactId>jacoco-maven-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-deploy-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-javadoc-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-jar-plugin</artifactId>
				<configuration>
					<archive>

				<artifactId>maven-surefire-plugin</artifactId>
			</plugin>
			<plugin>
				<groupId>org.jacoco</groupId>
				<artifactId>jacoco-maven-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-deploy-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-javadoc-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-jar-plugin</artifactId>
				<configuration>
					<archive>

					</excludes>
				</configuration>
			</plugin>
			<plugin>
				<groupId>org.jacoco</groupId>
				<artifactId>jacoco-maven-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-deploy-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-javadoc-plugin</artifactId>
			</plugin>
			<plugin>
				<artifactId>maven-jar-plugin</artifactId>
				<configuration>
					<archive>

    /** Delay in milliseconds after processing a URL */
    protected long delayMillisAfterProcessing = 0L;

    /** Delay in milliseconds when no URL is in the queue */
    protected long delayMillisAtNoUrlInQueue = 500L;

    /** Delay in milliseconds before processing a URL */
    protected long delayMillisBeforeProcessing = 0L;

    /** Delay in milliseconds for waiting for new URLs */

    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.
 */

    int NO_URL_IN_QUEUE = 4;

    /** Constant indicating that the crawler is waiting for new URLs. */
    int WAIT_NEW_URL = 8;

    /**
     * Introduces a delay based on the specified type.
     *
     * @param type the type of delay to be introduced
     */
    void delay(int type);

    <glob pattern="*.dist"/>
    <glob pattern="*.distz"/>
    <glob pattern="*.pkg"/>
    <glob pattern="*.bpk"/>
    <glob pattern="*.dump"/>
    <glob pattern="*.elc"/>
    <glob pattern="*.deploy"/>
  </mime-type>

  <mime-type type="application/oda">
    <glob pattern="*.oda"/>
  </mime-type>
  <mime-type type="application/oebps-package+xml">
    <glob pattern="*.opf"/>
  </mime-type>

 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay
 * mechanism for each of these stages.
 * </p>
 *
 * <p>
 * The class also provides a mechanism to ignore exceptions that may occur during the delay process.
 * If {@link #ignoreException} is set to true, any exceptions thrown during the delay will be caught
 * and ignored. Otherwise, they will be re-thrown as {@link CrawlerSystemException}.

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */