assertTrue(urlFilter.match("https://test.com/page"));
        assertTrue(urlFilter.match("ftp://files.com/document.pdf"));
        assertTrue(urlFilter.match("file:///home/user/file.txt"));
    }

    /**
     * Test match with complex URL patterns
     */
    public void test_match_complexUrls() {
        String sessionId = "test-session-010";
        urlFilter.init(sessionId);

/**
 * A response processor implementation that handles sitemaps.
 * It parses the response body as a SitemapSet, extracts URLs from the sitemaps,
 * and adds them as child URLs to be crawled.
 *
 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.
 * It then iterates through the sitemaps in the SitemapSet, extracts the URL
 * from each sitemap, and creates a new {@link RequestData} object for each URL.

        }
        return rule;
    }

    public static UserDictionary getUserDictionary(Environment env, Settings settings) {
        if (settings.get(USER_DICT_PATH_OPTION) != null && settings.get(USER_DICT_RULES_OPTION) != null) {
            throw new IllegalArgumentException(

 *
 * <p>
 * This class implements the {@link ReadingConverter} interface and provides a method to convert a given
 * Katakana string into a list of possible Alphabet readings. It uses a predefined mapping of Katakana
 * characters to their Alphabet equivalents, handling both single and double Katakana character combinations.
 * </p>
 *
 * <p>

 * and can handle GZIP compressed sitemaps.
 * The class provides methods to check if an input stream is a valid sitemap,
 * and to parse an input stream into a {@link SitemapSet} object.
 * It uses SAX parser for XML sitemaps and XML sitemap indexes,
 * and handles potential exceptions during parsing.
 * The class also includes inner classes for handling XML sitemap and sitemap index parsing.
 */

import org.codelibs.fess.crawler.log.LogType;

/**
 * Implementation of the {@link LogHelper} interface.
 * This class provides methods for logging various events during the crawling process.
 * It uses Log4j2 for logging.
 *
 * <p>
 * The class contains methods for logging different types of events, such as:
 * </p>
 * <ul>
 *   <li>Starting and finishing threads</li>
 *   <li>Starting and cleaning up crawling</li>

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

/**
 * DefaultContentsParser is an implementation of the ContentsParser interface.
 * It provides methods to parse search words, query logs, and documents into SuggestItem objects.
 *
 * <p>This class uses various utilities such as ReadingConverter, Normalizer, and SuggestAnalyzer
 * to process and analyze the input data.</p>
 *
 * <p>It also handles the exclusion of search words based on certain criteria and manages the

    - "*.x"

jobs:
  build:

    runs-on: ubuntu-latest
    env:
      PARENT_BRANCH: main

    steps:
    - uses: actions/checkout@v4
    - name: Set up JDK 21
      uses: actions/setup-java@v4
      with:
        java-version: '21'
        distribution: 'temurin'
    - uses: actions/cache@v4
      with:
        path: ~/.m2/repository
        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

            final File tempDir = File.createTempFile("crawlerDocRoot", "");
            tempDir.delete();
            tempDir.mkdirs();

            final StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");