public void test_run_noUrlsInQueue() throws Exception {
        when(urlQueueService.poll(anyString())).thenReturn(null);
        crawlerContext.setStatus(CrawlerStatus.RUNNING);
        crawlerContext.maxThreadCheckCount = 1; // Will exit after 1 check

        crawlerThread.run();

        verify(urlQueueService, times(1)).poll(anyString());
    }

    /**
     * Test run method with crawler status DONE.
     */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.service.impl;

import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;

        }

        // Should not throw exception for file with only comments
        assertNotNull(robotsTxt);
        // Everything should be allowed by default
        assertTrue(robotsTxt.allows("/anything", "AnyBot"));
    }

    public void testParse_onlyWhitespace() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots_only_whitespace.txt");

                }
            }

            // If pattern doesn't end with $, it implicitly matches anything after it
            if (pattern.isEmpty() || pattern.charAt(pattern.length() - 1) != '$') {
                // This makes "/fish" match "/fish", "/fishing", "/fish/", etc.
                // No need to add anything - the pattern naturally continues
            }

            return Pattern.compile(regex.toString());

    }

    public void test_directiveAllowsDefaultAllow() {
        // Test that paths are allowed by default
        Directive directive = new Directive("MyBot");

        assertTrue(directive.allows("/anything"));
        assertTrue(directive.allows("/public/"));
        assertTrue(directive.allows("/admin/"));
    }

    public void test_directiveAllowsPrecedence() {