/**
 * The Extractor interface defines methods for extracting text data from an input stream.
 * Implementations of this interface should provide the logic for extracting text and
 * optionally override the default weight value.
 */
public interface Extractor {

    /**
     * Extracts text data from the given input stream.
     *
     * @param in the input stream to extract text from

/**
 * @author shinsuke
 *
 */
public class FtpClientTest extends PlainTestCase {
    private static final int FTP_PORT = 10021;

    public FtpClient ftpClient;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("mimeTypeHelper", MimeTypeHelperImpl.class)//

     * @param params Additional parameters for the conversion.
     * @return The XContentBuilder with the URL filter data.
     * @throws IOException if the conversion fails.
     */
    @Override
    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
        builder.startObject();
        if (sessionId != null) {
            builder.field(SESSION_ID, sessionId);

    public SitemapsResponseProcessor() {
        super();
    }

    /**
     * Processes the given response data, extracting URLs from sitemaps.
     * @param responseData The response data.
     */
    @Override
    public void process(final ResponseData responseData) {
        final SitemapsHelper sitemapsHelper = crawlerContainer.getComponent("sitemapsHelper");

        super();
    }

    /**
     * Extracts text from the Word input stream.
     * @param in The input stream.
     * @param params The parameters.
     * @return The extracted data.
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {

 *
 */
public class TextExtractorTest extends PlainTestCase {
    private static final Logger logger = LogManager.getLogger(TextExtractorTest.class);

    public TextExtractor textExtractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("textExtractor", TextExtractor.class);

        final List<String> tagList = Analysis.parseWordList(environment, settings, "tags", s -> s);
        if (tagList != null) {
            posTags.addAll(tagList);
        }
    }

    @Override
    public TokenStream create(final TokenStream tokenStream) {
        final PartOfSpeechAttribute posAtt = tokenStream.addAttribute(PartOfSpeechAttribute.class);

        } else {
            keywordPath = null;
        }

        reloadInterval = settings.getAsTime("reload_interval", TimeValue.timeValueMinutes(1)).getMillis();
    }

    @Override
    public TokenStream create(final TokenStream tokenStream) {
        if (keywordPath == null) {
            return tokenStream;
        }

        }

        ignoreCase = settings.getAsBoolean("ignore_case", false);
        reloadInterval = settings.getAsTime("reload_interval", TimeValue.timeValueMinutes(1)).getMillis();
    }

    @Override
    public TokenStream create(final TokenStream tokenStream) {
        if (stopwordPath == null) {
            return tokenStream;
        }

        super();
    }

    /**
     * Extracts text from the Excel input stream.
     * @param in The input stream.
     * @param params The parameters.
     * @return The extracted data.
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {