*/
package org.codelibs.fess.crawler.transformer.impl;

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResultDataImpl;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

* [LastaFlute](https://github.com/lastaflute/lastaflute "LastaFlute"): Web-Framework
* [Lasta Job](https://github.com/lastaflute/lasta-job "Lasta Job"): Job-Scheduler
* [Fess Crawler](https://github.com/codelibs/fess-crawler "Fess Crawler"): Web-Crawler

 */
package org.codelibs.fess.crawler.service;

import java.util.List;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;
import org.codelibs.fess.crawler.util.OpenSearchCrawlerConfig;

/target/
/.settings/
/fess-crawler*/target/
/fess-crawler*/bin/
/fess-crawler*/mydbflute/
/fess-crawler*/.project
/fess-crawler*/.classpath
/fess-crawler*/.settings/
/fess-crawler*/dependency-reduced-pom.xml
/fess-crawler*/dbflute_crawler/log/*.log
/fess-crawler*/dbflute_crawler/schema/project-schema-robot.xml
/fess-crawler/phantomjsdriver.log
.project
.idea
*.iml
/.vscode

 */
package org.codelibs.fess.crawler.service.impl;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.codelibs.fess.crawler.entity.OpenSearchUrlFilter;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

    /**
     * Interval in milliseconds between crawler thread executions.
     * Used to control the rate at which new crawler threads are started
     * and the frequency of status checks.
     */
    protected long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;

    /**
     * Thread priority for crawler threads.
     * Defaults to normal thread priority.
     */

 */
package org.codelibs.fess.crawler.extractor.impl;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.InputStream;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.io.CloseableUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.transformer;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.extractor.Extractor;
import org.codelibs.fess.crawler.extractor.ExtractorFactory;
import org.codelibs.fess.exception.FessSystemException;

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client;

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.MaxLengthExceededException;
import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */

import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.http.Authentication;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.http.form.FormScheme;
import org.codelibs.fess.crawler.client.http.impl.AuthenticationImpl;