*   <li>Handling redirects</li>
 *   <li>Processing responses</li>
 *   <li>Handling exceptions during crawling and child URL processing</li>
 *   <li>Handling cases where no URL is in the queue</li>
 *   <li>Handling cases where no response processor or rule is found</li>
 *   <li>Handling system errors</li>
 * </ul>
 *
 * <p>
 * Each logging method checks the log level before logging the message.

                        "-u", "testuser2;test123", //
                        "-s", "public;/share;yes;no;no;testuser1", //
                        "-s", "users;/srv;no;no;no;testuser1,testuser2", //
                        "-s", "testuser1 private share;/testuser1;no;no;no;testuser1", //
                        "-s", "testuser2 private share;/testuser2;no;no;no;testuser2", //
                        "-g", "log level = 3");

        String query = "test query";
        String[] fields = {};
        List<String> keywords = SuggestUtil.getKeywords(query, fields);
        assertNotNull(keywords);
        assertEquals(0, keywords.size()); // No keywords because no fields match
    }

    @Test
    public void testGetKeywordsWithComplexQuery() {
        // Test with complex query structure
        String query = "(test AND query) OR (another AND search)";

        assertFalse(urlFilter.match("https://example.com/admin/dashboard"));
        assertFalse(urlFilter.match("https://other.com/page.html"));
    }

    /**
     * Test match with no patterns configured
     */
    public void test_match_noPatterns() {
        String sessionId = "test-session-009";
        urlFilter.init(sessionId);

        // Without any patterns, all URLs should match

     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */
    public int getCrawlDelay(final String userAgent) {
        final Directive directive = getMatchedDirective(userAgent);

        map.put("ナ", new String[] { "na" });
        map.put("ニ", new String[] { "ni" });
        map.put("ヌ", new String[] { "nu" });
        map.put("ネ", new String[] { "ne" });
        map.put("ノ", new String[] { "no" });

        map.put("ハ", new String[] { "ha" });
        map.put("ヒ", new String[] { "hi" });
        map.put("フ", new String[] { "hu", "fu" });
        map.put("ヘ", new String[] { "he" });

      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.

   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly

        String preloadDate = StringUtil.EMPTY;
        final byte[] bytes = new byte[preloadSize];
        try {
            if (bis.read(bytes) == -1) {
                throw new CrawlingAccessException("No sitemaps data.");
            }

            preloadDate = new String(bytes, Constants.UTF_8);
            if (preloadDate.indexOf("<urlset") >= 0) {
                // XML Sitemaps
                bis.reset();

      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.

   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly

    /**
     * Runs the crawling process in a separate thread.
     * This method fetches URLs from the queue, accesses content, processes responses,
     * and extracts child URLs until the crawling process is done or no more URLs are available.
     */
    @Override
    public void run() {
        log(logHelper, LogType.START_THREAD, crawlerContext);
        int threadCheckCount = 0;
        // set urlQueue to thread