#smart.deploy.mode.location = maihama_env.properties: lasta_di.smart.deploy.mode

# package for smart deploy target e.g. org.docksidestage.app
smart.package1 = org.codelibs.fess.crawler

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<!-- Crawler Thread -->
	<component name="crawlerThread" class="org.codelibs.fess.crawler.FessCrawlerThread" instance="prototype" >
	</component>

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/robotstxt.xml" />
	<include path="crawler/contentlength.xml" />
	<include path="crawler/mimetype.xml" />

	<component name="internalHttpClient" class="org.codelibs.fess.crawler.client.http.HcHttpClient"
		instance="prototype">
		<property name="connectionTimeout">15000</property>

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="robotsTxtHelper" class="org.codelibs.fess.crawler.helper.RobotsTxtHelper"
		instance="prototype">
	</component>

 */
package org.codelibs.fess.crawler.client;

import java.io.File;
import java.io.IOException;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;

	<modelVersion>4.0.0</modelVersion>
	<groupId>org.codelibs.fess</groupId>
	<artifactId>fess-crawler-parent</artifactId>
	<version>15.4.1-SNAPSHOT</version>
	<packaging>pom</packaging>
	<name>Fess Crawler Project</name>
	<description>Fess Crawler is Crawler Framework.</description>
	<url>https://github.com/codelibs/fess-crawler</url>
	<inceptionYear>2009</inceptionYear>
	<licenses>
		<license>

     */
    @Execute
    @Secured({ ROLE, ROLE + VIEW })
    public HtmlResponse index(final SearchForm form) {
        return asListHtml();
    }

    /**
     * Displays a paginated list of web crawler configurations.
     *
     * @param pageNumber the page number to display (optional)
     * @param form the search form containing filter criteria
     * @return HTML response with the web config list
     */

import org.codelibs.core.io.FileUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.ExtractException;
import org.codelibs.fess.crawler.extractor.Extractor;
import org.jodconverter.core.office.OfficeException;

 */
package org.codelibs.fess.crawler.rule.impl;

import java.io.File;
import java.io.InputStream;
import java.util.regex.Pattern;

import org.codelibs.core.io.CloseableUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.helper.SitemapsHelper;

import org.codelibs.core.timer.TimeoutTask;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.client.AccessTimeoutTarget;
import org.codelibs.fess.crawler.client.http.Authentication;
import org.codelibs.fess.crawler.client.http.RequestHeader;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.ExtractException;

import com.google.common.base.Charsets;