*
 * <p>Example usage:
 * <pre>
 *   Crawler crawler = new Crawler();
 *   crawler.addUrl("http://example.com/");
 *   crawler.execute();
 *   crawler.close();
 * </pre>
 */
public class Crawler implements Runnable, AutoCloseable {

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /**
     * Service for managing URL queues during crawling.
     */

        NameTrackingTransformer tracker = new NameTrackingTransformer();
        ResponseData responseData = new ResponseData();

        // Transform with different names
        tracker.setName("name1");
        tracker.transform(responseData);

        tracker.setName("name2");
        tracker.transform(responseData);

        tracker.setName("name3");
        tracker.transform(responseData);

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;
import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.crawler.util.OpenSearchCrawlerConfig;

import org.codelibs.core.misc.Pair;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;

import org.apache.logging.log4j.Logger;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * <p>

import org.codelibs.core.lang.ThreadUtil;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;

	<modelVersion>4.0.0</modelVersion>
	<artifactId>fess-crawler</artifactId>
	<packaging>jar</packaging>
	<name>Fess Crawler</name>
	<description>Fess Crawler is a crawler framework.</description>
	<parent>
		<groupId>org.codelibs.fess</groupId>
		<artifactId>fess-crawler-parent</artifactId>
		<version>15.3.0-SNAPSHOT</version>
		<relativePath>../pom.xml</relativePath>
	</parent>

 */
package org.codelibs.fess.crawler;

import java.io.File;

import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;

import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.lang.ThreadUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.ExecutionTimeoutException;
import org.codelibs.fess.crawler.exception.ExtractException;

/**
 * Extracts text content by executing an external command.
 */

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResultImpl;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.entity.UrlQueueImpl;
import org.codelibs.fess.crawler.extractor.ExtractorFactory;
import org.codelibs.fess.crawler.extractor.impl.TikaExtractor;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;