*/
    protected ResponseData getResponseData(final String uri, final boolean includeContent) {
        final ResponseData responseData = new ResponseData();
        try {
            responseData.setMethod(includeContent ? Constants.GET_METHOD : Constants.HEAD_METHOD);
            final String filePath = preprocessUri(uri);
            responseData.setUrl(filePath);

     * as a String.
     */
    protected Class<?> dataClass = null;

    @Override
    protected void storeData(final ResponseData responseData, final ResultData resultData) {
        final DOMParser parser = getDomParser();
        try (final InputStream in = responseData.getResponseBody()) {
            final InputSource is = new InputSource(in);
            if (responseData.getCharSet() != null) {

     */
    @Override
    public InetAddress[] resolve(final String host) throws UnknownHostException {
        return InetAddress.getAllByName(toAscii(host));
    }

    /**
     * Decodes the given host string using the specified encoding.
     *
     * @param host the host string to decode
     * @return the decoded host string
     */
    protected String decode(final String host) {
        if (host.indexOf('%') == -1) {

        // NOP
    }

    /**
     * Registers this rule with the {@link RuleManager}.
     *
     * @param index the index at which the rule should be registered
     */
    public void register(final int index) {
        final RuleManager ruleManager = crawlerContainer.getComponent("ruleManager");
        ruleManager.addRule(index, this);
    }

    @Override
    public String getRuleId() {
        return ruleId;
    }

     * Field name for ID.
     */
    public static final String ID = "id";

    /**
     * Field name for session ID.
     */
    public static final String SESSION_ID = "sessionId";

    /**
     * Field name for rule ID.
     */
    public static final String RULE_ID = "ruleId";

    /**
     * Field name for URL.
     */
    public static final String URL = "url";

    /**

    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            final BufferedInputStream bis = new BufferedInputStream(in);
            final String enc = getEncoding(bis);

public class SmbAuthenticationHolderTest extends PlainTestCase {

    public void test_get() {
        final SmbAuthenticationHolder smbAuthenticationHolder = new SmbAuthenticationHolder();
        final SmbAuthentication hogeAuth = new SmbAuthentication();
        hogeAuth.setServer("hoge");
        smbAuthenticationHolder.add(hogeAuth);
        final SmbAuthentication fugaAuth = new SmbAuthentication();
        fugaAuth.setServer("fuga");

        final BufferedInputStream bis = new BufferedInputStream(in);
        final String encoding = htmlXpathExtractor.getEncoding(bis);
        CloseableUtil.closeQuietly(bis);
        assertEquals("UTF-8", encoding);
    }

    public void test_getEncoding_sjis() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/test_sjis.html");
        final BufferedInputStream bis = new BufferedInputStream(in);

    private final ThreadLocal<XPathAPI> xpathAPI = new ThreadLocal<>();

    /**
     * Creates a new HtmlExtractor instance.
     */
    public HtmlExtractor() {
        super();
    }

    @Override
    protected ExtractData createExtractData(final String content) {
        final DOMParser parser = getDomParser();
        try (final Reader reader = new StringReader(content)) {

     */
    @Override
    public String generateType3Msg(final String username, final String password, final String domain, final String workstation,
            final String challenge) throws NTLMEngineException {
        Type2Message type2Message;
        try {
            type2Message = new Type2Message(Base64.getDecoder().decode(challenge));
        } catch (final IOException exception) {