urlFilter.addInclude("http://example.com/.*");
        urlFilter.addInclude("http://test.com/.*");

        final String sessionId = "id1";
        urlFilter.init(sessionId);

        assertTrue(urlFilter.match("http://example.com/"));
        assertTrue(urlFilter.match("http://example.com/a"));
        assertTrue(urlFilter.match("http://test.com/"));
        assertTrue(urlFilter.match("http://test.com/a"));
    }

        url = "http://hoge/index.html#hoge";
        assertEquals("http://hoge/index.html", htmlTransformer.normalizeUrl(url));

        url = "http://hoge/index.html#";
        assertEquals("http://hoge/index.html", htmlTransformer.normalizeUrl(url));

        url = "http://hoge/index.html;jsessionid=hoge";
        assertEquals("http://hoge/index.html", htmlTransformer.normalizeUrl(url));

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client.http.conn;

import java.io.UnsupportedEncodingException;
import java.net.IDN;
import java.net.InetAddress;
import java.net.URLDecoder;
import java.net.UnknownHostException;

import org.apache.http.conn.DnsResolver;

/**
 * The {@code IdnDnsResolver} class implements the {@code DnsResolver} interface

import java.util.List;
import java.util.Map;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScheme;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.client.AuthCache;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;

        assertNull(dataService.getAccessResult("id1", "http://www.id1.com/"));
        assertNotNull(dataService.getAccessResult("id2", "http://www.id2.com/"));

        dataService.store(accessResult1);
        assertNotNull(dataService.getAccessResult("id1", "http://www.id1.com/"));

        dataService.deleteAll();

        assertNull(dataService.getAccessResult("id1", "http://www.id1.com/"));

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components>
    <include path="crawler_opensearch.xml"/>

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client.http;

import org.apache.http.auth.AuthScheme;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;

/**
 * The Authentication interface provides methods to retrieve authentication details
 * required for HTTP client authentication.
 */
public interface Authentication {

    /**

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client.http.ntlm;

import org.apache.http.auth.AuthScheme;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.impl.auth.NTLMScheme;
import org.apache.http.protocol.HttpContext;

/**
 * This class is AuthSchemeFactory implementation for NTLM.
 *
 * @author shinsuke
 *
 */

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/transformer_basic.xml"/>

 * Represents a request data for crawling.
 * This class encapsulates the HTTP method, URL, and weight associated with a crawling request.
 */
public class RequestData {
    /**
     * HTTP methods supported for crawling requests.
     */
    public enum Method {
        /** HTTP GET method. */
        GET,
        /** HTTP POST method. */
        POST,
        /** HTTP HEAD method. */
        HEAD;
    }