public void test_run_noUrlsInQueue() throws Exception {
        when(urlQueueService.poll(anyString())).thenReturn(null);
        crawlerContext.setStatus(CrawlerStatus.RUNNING);
        crawlerContext.maxThreadCheckCount = 1; // Will exit after 1 check

        crawlerThread.run();

        verify(urlQueueService, times(1)).poll(anyString());
    }

    /**
     * Test run method with crawler status DONE.
     */

        }

        // Should not throw exception for file with only comments
        assertNotNull(robotsTxt);
        // Everything should be allowed by default
        assertTrue(robotsTxt.allows("/anything", "AnyBot"));
    }

    public void testParse_onlyWhitespace() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots_only_whitespace.txt");

                }
            }

            // If pattern doesn't end with $, it implicitly matches anything after it
            if (pattern.isEmpty() || pattern.charAt(pattern.length() - 1) != '$') {
                // This makes "/fish" match "/fish", "/fishing", "/fish/", etc.
                // No need to add anything - the pattern naturally continues
            }

            return Pattern.compile(regex.toString());