userManager.save(ftpUser);
        }

        FtpServer server = factory.createServer();
        server.start();
        return server;
    }

    public void test_doGet_root_dir() throws FtpException {
        FtpServer server = null;
        try {
            String username = "testuser";
            String password = "testpass";
            server = startFtpServer(FTP_PORT, username, password);

            urlSet.add("http://example" + i + ".com/robots.txt");
        }
        assertEquals(10000, urlSet.size());

        // Add one more should maintain size at 10000 (LRU eviction)
        urlSet.add("http://overflow.com/robots.txt");
        assertEquals(10000, urlSet.size());
        assertTrue(urlSet.contains("http://overflow.com/robots.txt"));

    }

    public void test_doGet() {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final ResponseData responseData = httpClient.doGet(url);
            assertEquals(200, responseData.getHttpStatusCode());
        } finally {
            server.stop();
        }
    }

```

## Supported Protocols and Formats

### Protocols
- **HTTP/HTTPS**: Full web crawling support with cookies, authentication, redirects
- **File System**: Local and network file system access
- **FTP**: FTP server crawling with authentication
- **SMB/CIFS**: Windows network shares
- **Storage**: Cloud storage systems (MinIO, S3-compatible)

### Content Formats

#### Office Documents

        assertTrue(exceptions.isEmpty());
        assertEquals(threadCount * objectsPerThread, createdComponents.size());

        // Check all components are unique (for prototype)
        for (int i = 0; i < createdComponents.size(); i++) {
            for (int j = i + 1; j < createdComponents.size(); j++) {
                assertNotSame(createdComponents.get(i), createdComponents.get(j));
            }
        }
    }

            dataService.delete(sessionId);
        } finally {
            server.stop();
        }
    }

    public void test_execute_xmlSitemaps() throws Exception {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;

    }

    /**
     * Gets the buffer size for data transfers.
     *
     * @return The buffer size
     */
    public int getBufferSize() {
        return bufferSize;
    }

    /**
     * Sets the buffer size for data transfers.
     *
     * @param bufferSize The buffer size
     */
    public void setBufferSize(final int bufferSize) {

            dataService.delete(sessionId);
        } finally {
            server.stop();
        }
    }

    public void test_execute_xmlSitemaps() throws Exception {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;

    private static boolean removeLastDuplication(final StringBuilder buf, final int size, final boolean isSpace,
            final Set<String> termCache) {
        final String target = rightString(buf, size);
        if (!termCache.contains(target)) {
            termCache.add(target);
            return isSpace;
        }
        buf.setLength(buf.length() - size);
        if (!isSpace && !isLastSpaceChar(buf)) {

        assertNotNull(keywords);
        assertEquals(2, keywords.size());
        assertTrue(keywords.contains("test"));
        assertTrue(keywords.contains("query"));

        // Test with OR operator
        List<String> orKeywords = SuggestUtil.getKeywords("test OR query", fields);
        assertNotNull(orKeywords);
        assertEquals(2, orKeywords.size());

        // Test with invalid query