assertEquals("ftp://localhost:10021/dir2", urlList.get(1));
            assertEquals("ftp://localhost:10021/text 3.txt", urlList.get(2));
            assertEquals("ftp://localhost:10021/text1.txt", urlList.get(3));
            assertEquals("ftp://localhost:10021/text2.txt", urlList.get(4));
        } finally {
            if (server != null) {
                server.stop();
            }
        }

mvn license:format

# Run static analysis
mvn spotbugs:check
```

### Running Tests

```bash
# Run all tests
mvn test

# Run specific test class
mvn test -Dtest=CrawlerTest

# Run specific test method
mvn test -Dtest=CrawlerTest#test_execute_web

# Run tests for specific module
mvn test -pl fess-crawler
```

## Examples

### Web Crawling with Custom Rules

        //final ExtractData text = extractor.getText(new ByteArrayInputStream(FileUtils.readFileToByteArray(new File(filePath))), params);
        final ExtractData text = extractor.getText(new ByteArrayInputStream(testStr.getBytes()), params);
        assertEquals(content, text.getContent());
    }

    // TODO other tests

    static class TestApiExtractorServer {
        private Server server;

import org.opensearch.core.common.Strings;
import org.opensearch.transport.client.Client;

import com.ibm.icu.text.Transliterator;

/**
 * AnalyzerConverter is a class that implements the ReadingConverter interface.
 * It is responsible for converting text using specified language analyzers.
 * The class uses a Transliterator to convert between Hiragana and Katakana.
 *
 * <p>Constructor:
 * <ul>

        String text = "test text";
        String field = "content";

        List<String> results = converter.convert(text, field, "en");

        assertNotNull(results);
        // Results may vary based on analyzer configuration
    }

    @Test
    public void testConvertWithMultipleLanguages() throws IOException {
        // Test convert with multiple languages
        String text = "test text";

        }

        /**
         * Normalizes the given text based on the specified field.
         *
         * @param text  the text to normalize
         * @param field the field to use for normalization
         * @param dummy additional parameters (not used)
         * @return the normalized text
         */
        @Override
        public String normalize(final String text, final String field, final String... dummy) {

            }
        }
    }

    /**
     * Extracts text from an embedded file using the appropriate extractor.
     * @param filename the filename of the embedded file
     * @param embeddedFile the embedded file to extract text from
     * @param writer the writer to append extracted text to
     */

        // nothing
    }

    @Override
    public String normalize(final String text, final String field, final String... langs) {
        final char[] chars = new char[text.length()];
        for (int i = 0; i < chars.length; i++) {
            final char c = text.charAt(i);
            if (c >= 'ａ' && c <= 'ｚ') {
                chars[i] = (char) (c - 'ａ' + 'a');

/**
 * The Extractor interface defines methods for extracting text data from an input stream.
 * Implementations of this interface should provide the logic for extracting text and
 * optionally override the default weight value.
 */
public interface Extractor {

    /**
     * Extracts text data from the given input stream.
     *
     * @param in the input stream to extract text from

    }

    /**
     * Extracts a date string from the received header text.
     *
     * @param text the received header text
     * @return the date string starting from the day of week, or null if not found
     */
    private static String getDateString(final String text) {
        for (final String dow : DAY_OF_WEEK) {
            final int i = text.lastIndexOf(dow);
            if (i != -1) {