import com.ibm.icu.text.Transliterator;

/**
 * Converts Katakana characters to their corresponding Alphabet representations.
 *
 * <p>
 * This class implements the {@link ReadingConverter} interface and provides a method to convert a given
 * Katakana string into a list of possible Alphabet readings. It uses a predefined mapping of Katakana
 * characters to their Alphabet equivalents, handling both single and double Katakana character combinations.

      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a

      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a

    protected String outputEncoding = Constants.UTF_8;

    /** Map of input file extensions to output extensions. */
    private final Map<String, String> extensionMap = new HashMap<>();

    /** Map of output extensions to their corresponding extractors. */
    private final Map<String, Extractor> extractorMap = new HashMap<>();

    /**
     * Constructs a new JodExtractor and initializes the extension and extractor mappings.
     */

 * </p>
 *
 * <p>
 * The reader supports limiting the number of documents read based on a percentage of the total documents
 * or a fixed number. It also allows filtering documents based on their size, using the {@code limitOfDocumentSize}
 * parameter.
 * </p>
 *
 * <p>
 * The reader uses a queue to buffer documents read from Elasticsearch, and it retries failed requests
 * up to a maximum number of times.

 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    /** Map of user agent patterns to their corresponding directives. */
    protected final Map<Pattern, Directive> directiveMap = new LinkedHashMap<>();

    /** List of sitemap URLs found in the robots.txt file. */

        this.daemon = daemon;
    }

    /**
     * Runs the crawling process in a separate thread.
     * This method initializes the crawler context, creates and starts crawler threads,
     * and waits for their completion.
     */
    @Override
    public void run() {
        // context
        crawlerContext.urlFilter = urlFilter;
        crawlerContext.ruleManager = ruleManager;