# Fess Crawler

[![Java CI with Maven](https://github.com/codelibs/fess-crawler/actions/workflows/maven.yml/badge.svg)](https://github.com/codelibs/fess-crawler/actions/workflows/maven.yml)
[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)

        urlFilter.addInclude("https://example.com/.*");
        urlFilter.addInclude("https://test.com/.*");
        urlFilter.addInclude(".*\\.pdf$");

        assertTrue(urlFilter.match("https://example.com/"));
        assertTrue(urlFilter.match("https://test.com/page"));
        assertTrue(urlFilter.match("https://any.com/document.pdf"));
        assertFalse(urlFilter.match("https://other.com/page.html"));
    }

        assertEquals("http://localhost/home#section1", HcHttpClient.constructRedirectLocation("http://localhost/", "/home#section1"));
        assertEquals("http://localhost/newpage", HcHttpClient.constructRedirectLocation("http://localhost", "newpage"));
        assertEquals("http://localhost/newpage", HcHttpClient.constructRedirectLocation("http://localhost/path/", "../newpage"));

https://twitter.com/Mint_Lee/status/626564933350600704

@withnewsjp
デビューは23年前、アイドル冬の時代から変わらぬスタイル

>制服向上委員会とは何者か？「自民倒そう」　民主時代は「野田ダメ」 - ウィズニュース http://t.co/qtVCWlrmy8 #withnews #制服向上委員会 http://t.co/6R1CL9LwfT
https://twitter.com/withnewsjp/status/626512633483522049

@lwnish
今宵 http://t.co/ca7DKqPWVU
https://twitter.com/lwnish/status/626695622242926593

@Newsweek_JAPAN

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>org.codelibs.fess</groupId>
	<artifactId>fess-crawler-parent</artifactId>
	<version>15.3.0-SNAPSHOT</version>
	<packaging>pom</packaging>
	<name>Fess Crawler Project</name>

        }

        /**
         * Sets the HTTP method for the request.
         * @param method The HTTP method.
         * @return The current RequestDataContext instance.
         */
        public RequestDataContext method(final Method method) {
            data.setMethod(method);
            return this;
        }

        /**
         * Sets the HTTP method to GET.

            final String preloadDate = new String(bytes, Constants.UTF_8);
            if (preloadDate.indexOf("<urlset") >= 0 || preloadDate.indexOf("<sitemapindex") >= 0 || preloadDate.startsWith("http://")
                    || preloadDate.startsWith("https://")) {
                // XML Sitemaps
                return true;
            }
            // gz
            bis.reset();
            return isValid(new GZIPInputStream(bis), false);

        assertEquals("http://example.com/.*", domainFilter.cachedIncludeSet.toArray()[0]);
        assertEquals("http://example.com/.*", domainFilter.cachedExcludeSet.toArray()[0]);

        domainFilter.processUrl("https://test.com");

        assertEquals(2, domainFilter.cachedIncludeSet.size());
        assertEquals(2, domainFilter.cachedExcludeSet.size());

    <root-XML localName="kml"/>
    <root-XML namespaceURI="http://www.opengis.net/kml/2.2" localName="kml"/>
    <root-XML namespaceURI="http://earth.google.com/kml/2.0" localName="kml"/>
    <root-XML namespaceURI="http://earth.google.com/kml/2.1" localName="kml"/>
    <root-XML namespaceURI="http://earth.google.com/kml/2.2" localName="kml"/>
    <acronym>KML</acronym>

        responseData.setHttpStatusCode(200);
        responseData.setUrl("http://example.com/");
        responseData.setMimeType("text/html");
        return responseData;
    }

    private ResponseData getTestData2() {
        final ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        responseData.setUrl("https://example.com/");
        responseData.setMimeType("text/html");