responseData.setHttpStatusCode(Constants.OK_STATUS_CODE);
            responseData.setCharSet(Constants.UTF_8);
            responseData.setLastModified(file.getTimestamp().getTime());

            // check file size
            responseData.setContentLength(file.getSize());
            checkMaxContentLength(responseData);

            if (file.getUser() != null) {

                    startedCrawlerNum++;
                    activeCrawlerNum++;
                    ThreadUtil.sleep(crawlingExecutionInterval);
                    continue;
                }

                // check status
                for (int i = 0; i < startedCrawlerNum; i++) {
                    if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE

    }

    /**
     * Returns data as HTML content of String.
     *
     */
    @Override
    public Object getData(final AccessResultData<?> accessResultData) {
        // check transformer name
        if (!getName().equals(accessResultData.getTransformerName())) {
            throw new CrawlerSystemException(

        if (idx >= 0) {
            hostUrl = url.substring(0, idx);
        } else {
            hostUrl = url;
        }
        final String robotTxtUrl = hostUrl + "/robots.txt";

        // check url
        if (crawlerContext.getRobotsTxtUrlSet().contains(robotTxtUrl)) {
            if (logger.isDebugEnabled()) {
                logger.debug("{} is already visited.", robotTxtUrl);
            }

    //                                                                              ======
    @Override
    protected int delegateSelectCountUniquely(final ConditionBean cb) {
        // #pending check response and cast problem
        final SearchRequestBuilder builder = client.prepareSearch(asEsIndex());
        final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
        if (esCb.getPreference() != null) {

    //                                                                              ======
    @Override
    protected int delegateSelectCountUniquely(final ConditionBean cb) {
        // #pending check response and cast problem
        final SearchRequestBuilder builder = client.prepareSearch(asEsIndex());
        final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
        if (esCb.getPreference() != null) {

                }
                throw new ChildUrlsException(requestDataSet, this.getClass().getName() + "#getResponseData");
            }
            // check file size
            responseData.setContentLength(statObject.size());
            checkMaxContentLength(responseData);

            responseData.setHttpStatusCode(Constants.OK_STATUS_CODE);

    }

    public CrawlJob hotThread(final int hotThreadInterval) {
        this.hotThreadInterval = hotThreadInterval;
        return this;
    }

    @Override
    public String execute() {
        //   check # of crawler processes
        final int maxCrawlerProcesses = ComponentUtil.getFessConfig().getJobMaxCrawlerProcessesAsInteger();
        if (maxCrawlerProcesses > 0) {
            final int runningJobCount = getRunningJobCount();

     *
     * @return XML content of String.
     */
    @Override
    public Object getData(final AccessResultData<?> accessResultData) {
        if (dataClass == null) {
            // check transformer name
            if (!getName().equals(accessResultData.getTransformerName())) {
                throw new CrawlerSystemException("Transformer is invalid. Use " + accessResultData.getTransformerName()

            final GetResponse response = getClient().get(c -> c.prepareGet(index, id).execute());
            return response.isExists();
        } catch (final Exception e) {
            throw new EsAccessException("Failed to check if " + sessionId + ":" + url + " exists.", e);
        }
    }

    public int getCount(final Consumer<SearchRequestBuilder> callback) {
        final TotalHits totalHits = getClient().get(c -> {