*/
    public RobotsTxt parse(final InputStream stream, final String charsetName) {
        if (!enabled) {
            return null;
        }

        try {
            @SuppressWarnings("resource")
            final BufferedReader reader = new BufferedReader(new InputStreamReader(new BOMInputStream(stream), charsetName));

            String line;
            final RobotsTxt robotsTxt = new RobotsTxt();

     * @param threadPool The ExecutorService for thread pooling.
     */
    public Suggester(final Client client, final SuggestSettings settings, final ReadingConverter readingConverter,
            final ReadingConverter contentsReadingConverter, final Normalizer normalizer, final SuggestAnalyzer analyzer,
            final ExecutorService threadPool) {
        this.client = client;
        suggestSettings = settings;

        final byte[] bytes = "".getBytes();
        final InputStream in = new ByteArrayInputStream(bytes);

        try {
            sitemapsHelper.parse(in);
            fail();
        } catch (final CrawlingAccessException e) {
            // NOP
        }
    }

    public void test_parseXmlSitemaps_invalid2() {
        final byte[] bytes = "test".getBytes();

     * @param settingsId Settings ID
     */
    protected ElevateWordSettings(final SuggestSettings settings, final Client client, final String settingsIndexName,
            final String settingsId) {
        arraySettings = new ArraySettings(settings, client, settingsIndexName, settingsId) {
            @Override
            protected String createArraySettingsIndexName(final String settingsIndexName) {
                return settingsIndexName + "_elevate";

        // 同時実行数
        final int numTasks = 100;
        // インターバル
        final Long waittime = 100L;

        CrawlingParameterUtil.setUrlQueue(new UrlQueueImpl());
        final UrlQueue q = CrawlingParameterUtil.getUrlQueue();
        for (int i = 0; i < numTasks; i++) {
            q.setUrl("http://example.com");
        }

        final HostIntervalController controller = new HostIntervalController();

        ruleManager.addRule(fileRule);
    }

    public void test_getRule() {
        final Rule rule = ruleManager.getRule(new ResponseData());
        assertNotNull(rule);
        assertEquals("fileRule", rule.getRuleId());
    }

    public void test_getRule_sitemaps1() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml");

    public void test_transform_text() throws Exception {
        final byte[] data = new String("xyz").getBytes();
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("file:/test.txt");
        responseData.setCharSet(Constants.UTF_8);
        responseData.setResponseBody(data);
        responseData.setMimeType("text/plain");
        final ResultData resultData = textTransformer.transform(responseData);

public class CharTypeFilterFactory extends AbstractTokenFilterFactory {

    private final boolean alphabetic;

    private final boolean digit;

    private final boolean letter;

    public CharTypeFilterFactory(final IndexSettings indexSettings, final Environment environment, final String name,
            final Settings settings) {
        super(indexSettings, name, settings);

        // nothing
    }

    private final List<ReadingConverter> converters = new ArrayList<>();

    @Override
    public void init() throws IOException {
        for (final ReadingConverter converter : converters) {
            converter.init();
        }
    }

    @Override
    public List<String> convert(final String text, final String field, final String... lang) throws IOException {

        try {
            mimeTypes = MimeTypesFactory.create(MIME_TYPES_RESOURCE_NAME);
        } catch (final Exception e) {
            throw new CrawlerSystemException("Could not initialize MimeTypeHelper.", e);
        }
    }

    @Override
    public String getContentType(final InputStream is, final String filename) {
        final Map<String, String> params = new HashMap<>();
        params.put(ExtractData.RESOURCE_NAME_KEY, filename);