import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.ExtractException;
import org.codelibs.fess.crawler.extractor.Extractor;
import org.codelibs.fess.crawler.util.TextUtil;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.helper;

import java.util.HashMap;
import java.util.Map;

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * Helper class for managing content length limits based on MIME types.

     */
    public String cronExpression;

    /**
     * Script type of the scheduled job.
     */
    public String scriptType;

    /**
     * Whether the scheduled job is a crawler job.
     */
    public String crawler;

    /**
     * Whether logging is enabled for the scheduled job.
     */
    public String jobLogging;

    /**
     * Whether the scheduled job is available.
     */

            false, "Boolean", 0, 0, null, null, false, null, null, null, null, null, false);
    protected final ColumnInfo _columnCrawler = cci("crawler", "crawler", null, null, Boolean.class, "crawler", null, false, false, false,
            "Boolean", 0, 0, null, null, false, null, null, null, null, null, false);

 */
package org.codelibs.fess.crawler.transformer.impl;

import java.util.Map;

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.entity.AccessResultDataImpl;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval.impl;

import java.util.Map;

import org.codelibs.core.lang.ThreadUtil;

/**
 * Default implementation of the IntervalController.
 * This class provides a default way to manage delays between crawler operations.
 * It allows setting delays before processing, after processing, when no URLs are in the queue,

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.service;

import java.util.List;

import org.codelibs.fess.crawler.entity.UrlQueue;

/**
 * Service interface for managing URL queues.
 * Provides methods for adding, retrieving, and managing URLs within a crawling session.
 *

 */
package org.codelibs.fess.crawler.service.impl;

import static org.codelibs.opensearch.runner.OpenSearchRunner.newConfigs;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.codelibs.fess.crawler.client.FesenClient;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;

import org.apache.logging.log4j.Logger;
import org.codelibs.core.io.InputStreamUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.ExtractException;

/**
 * Abstract base class for XML extractors.

	<include path="fess_query.xml"/>
	<include path="fess_rankfusion.xml"/>
	<include path="fess_score.xml"/>
	<include path="fess_sso.xml"/>
	<include path="fess_thumbnail.xml"/>

	<include path="crawler/client.xml" />
	<include path="crawler/mimetype.xml" />

	<component name="accessTokenHelper" class="org.codelibs.fess.helper.AccessTokenHelper">
	</component>
	<component name="activityHelper" class="org.codelibs.fess.helper.ActivityHelper">