while (!crawlerRunning) {
            ThreadUtil.sleepQuietly(crawlerWaitMillis);
        }
    }

    /**
     * Applies delay based on the configured interval rules.
     * This method calculates the appropriate delay for the current time
     * and applies it by sleeping the current thread.
     */
    public void delayByRules() {
        final long delay = getDelay();
        if (delay > 0) {

 * and processes them to create thumbnail images based on configured dimensions
 * and format settings.
 *
 * <p>The generator validates image MIME types, processes image data through
 * ImageIO operations, and applies scaling and cropping to generate thumbnails
 * that meet the specified size requirements.</p>
 *
 */
public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {

    }

    /**
     * Adds a boost function with a filter to modify document scoring during search.
     * This method adds a boost function that applies only to documents matching the filter.
     *
     * @param filter the query filter to determine which documents the boost applies to
     * @param scoreFunction the score function to add for boosting
     */

 *
 * <p>The updater performs the following key operations:
 * <ul>
 * <li>Retrieves crawled documents from the data service</li>
 * <li>Transforms document data using appropriate transformers</li>
 * <li>Applies document boosting rules and click/favorite count enhancements</li>
 * <li>Sends processed documents to the search engine for indexing</li>
 * <li>Manages cleanup of processed crawler session data</li>
 * </ul>
 *

            }
        } catch (final Exception e) {
            logger.warn("Failed to initialize TikaExtractor.", e);
        }
    }

    /**
     * Processes and normalizes a document title.
     * Applies text normalization using configured space characters and returns
     * a clean title suitable for indexing.
     *
     * @param responseData the response data from crawling (not currently used)

     * A directive consists of a user agent, crawl delay, allowed paths, and disallowed paths.
     */
    public static class Directive {
        /** The user agent string this directive applies to. */
        private final String userAgent;

        /** The crawl delay in seconds for this directive. */
        private int crawlDelay;

        /** The list of allowed path patterns for this directive. */

        if (logger.isDebugEnabled()) {
            logger.debug("Total documents added: {}", documentSize.get());
        }

    }

    /**
     * Processes a document through the ingest pipeline.
     * Applies all available ingesters to transform the document data.
     *
     * @param paramMap the data store parameters
     * @param dataMap the document data to process
     * @return the processed document data
     */

        }

        return url;
    }

    /**
     * Extracts and processes the site path from a URL with proper encoding handling.
     * Removes protocol, query parameters, and applies URL decoding based on encoding settings.
     *
     * @param u the URL string to process
     * @param encoding the character encoding to use for URL decoding
     * @return the processed site path, abbreviated if necessary

Licenses are intended to guarantee your freedom to share and change
free software--to make sure the software is free for all its users.

  This license, the Lesser General Public License, applies to some
specially designated software packages--typically libraries--of the
Free Software Foundation and other authors who decide to use it.  You
can use it too, but we suggest you first think carefully about whether

        });
    }

    /**
     * Generates a hashed ID from the provided URL ID string.
     * Encodes special characters using URL encoding or Base64 encoding as needed,
     * then applies a message digest algorithm to create a unique hash.
     *
     * @param urlId the URL ID string to generate a hash for
     * @return a hashed ID string generated from the input URL ID
     */