* @param method the HTTP method
     */
    public void setMethod(final Method method) {
        this.method = method;
    }

    /**
     * Sets the HTTP method for this request using a string value.
     * Defaults to GET if the method is not recognized.
     * @param method the HTTP method as a string
     */
    public void setMethod(final String method) {
        if (Constants.GET_METHOD.equals(method)) {

        final BufferedInputStream bis = new BufferedInputStream(in);
        final String encoding = htmlExtractor.getEncoding(bis);
        CloseableUtil.closeQuietly(bis);
        assertEquals("UTF-8", encoding);
    }

    public void test_getEncoding_sjis() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/test_sjis.html");
        final BufferedInputStream bis = new BufferedInputStream(in);

        accessResultDataImpl.setTransformerName("transformer");

        try {
            final Object obj = xmlTransformer.getData(accessResultDataImpl);
            fail();
        } catch (final CrawlerSystemException e) {}
    }

    public void test_getData_nullData() throws Exception {
        final String value = "<?xml version=\"1.0\"?>\n"//
                + "<doc>\n"//

        super();
    }

    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }

        final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
        final ExtractorFactory extractorFactory = getExtractorFactory();

    }

    @Override
    public SuggestWriterResult write(final Client client, final SuggestSettings settings, final String index, final SuggestItem[] items,
            final boolean update) {
        throw new UnsupportedOperationException("not yet.");
    }

    @Override
    public SuggestWriterResult delete(final Client client, final SuggestSettings settings, final String index, final String id) {

            return tempDir;
        } catch (final Exception e) {
            throw new CrawlerSystemException(e);
        }
    }

    private static void generateContents(final File dir, final int count) throws Exception {
        if (count <= 0) {
            return;
        }

        final String content = getHtmlContent(count);

        final File indexFile = new File(dir, "index.html");

    }

    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }

        final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
        final ExtractorFactory extractorFactory = getExtractorFactory();
        final StringBuilder buf = new StringBuilder(1000);

        final List<Pattern> excludeList = getUrlFilterService().getExcludeUrlPatternList(sessionId);

        if (!includeList.isEmpty()) {
            boolean match = false;
            for (final Pattern pattern : includeList) {
                final Matcher matcher = pattern.matcher(url);
                if (matcher.matches()) {
                    match = true;

    private static final char HANKAKU_KATAKANA_FIRST_CHAR = HANKAKU_KATAKANA[0];

    private static final char HANKAKU_KATAKANA_LAST_CHAR = HANKAKU_KATAKANA[HANKAKU_KATAKANA.length - 1];

    @Override
    public String normalize(final String s, final String field, final String... langs) {
        if (s.length() == 0) {
            return s;
        }
        if (s.length() == 1) {

    }

    public static String normalizeText(final String str, final int initialCapacity, final int maxAlphanumTermSize,
            final int maxSymbolTermSize, final boolean removeDuplication) {
        if (str == null) {
            return StringUtil.EMPTY;
        }
        try (final Reader reader = new StringReader(str)) {