responseData.setExecutionTime(SystemUtil.currentTimeMillis() - startTime);
                            responseData.setParentUrl(urlQueue.getParentUrl());
                            responseData.setSessionId(crawlerContext.sessionId);

                            if (responseData.getRedirectLocation() == null) {
                                log(logHelper, LogType.PROCESS_RESPONSE, crawlerContext, urlQueue, responseData);

    public void add(final String sessionId, final String url) {
        if (exists(sessionId, url)) {
            return;
        }
        final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
        urlQueue.setSessionId(sessionId);
        urlQueue.setUrl(url);
        urlQueue.setCreateTime(System.currentTimeMillis());
        urlQueue.setLastModified(0L);
        urlQueue.setDepth(0);

            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());

            final Crawler crawler1 = crawlerContainer.getComponent("crawler");
            crawler1.setSessionId(crawler1.getSessionId() + "1");
            crawler1.setBackground(true);
            ((UrlFilterImpl) crawler1.urlFilter).setIncludeFilteringPattern("$1$2$3.*");
            crawler1.addUrl(url1);

            if (dataMap.containsKey(Constants.SESSION_ID)) {
                responseData.setSessionId((String) dataMap.get(Constants.SESSION_ID));
            } else {
                responseData.setSessionId((String) paramMap.get(Constants.CRAWLING_INFO_ID));
            }

            final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);

            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());

            final Crawler crawler1 = container.getComponent("crawler");
            crawler1.setSessionId(crawler1.getSessionId() + "1");
            crawler1.setBackground(true);
            ((UrlFilterImpl) crawler1.urlFilter).setIncludeFilteringPattern("$1$2$3.*");
            crawler1.addUrl(url1);

            // create crawler
            final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
            crawler.setSessionId(sid);
            sessionIdList.add(sid);

            final String urlsStr = webConfig.getUrls();
            if (StringUtil.isBlank(urlsStr)) {
                logger.warn("No target urls. Skipped");

    }


    /**
     * @param sessionId
     *            the sessionId to set
     */
    @Override
    public final void setSessionId ( long sessionId ) {
        this.sessionId = sessionId;
        if ( this.next != null ) {
            this.next.setSessionId(sessionId);
        }
    }


    /**
     * {@inheritDoc}
     *

                    }).orElse(null);//TODO
                    if (crawlingInfo == null) {
                        crawlingInfo = new CrawlingInfo();
                        crawlingInfo.setSessionId(list.get(0));
                        crawlingInfo.setCreatedTime(formatter.parse(list.get(1)).getTime());
                        crawlingInfoBhv.insert(crawlingInfo, op -> op.setRefreshPolicy(Constants.TRUE));

            }
            responseData.setExecutionTime(systemHelper.getCurrentTimeAsLong() - startTime);
            responseData.setSessionId(crawlingInfoId);

            final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);
            final Rule rule = ruleManager.getRule(responseData);
            if (rule == null) {

                    responseData.setExecutionTime(systemHelper.getCurrentTimeAsLong() - startTime);
                    responseData.setParentUrl(urlQueue.getParentUrl());
                    responseData.setSessionId(crawlerContext.getSessionId());
                    responseData.setHttpStatusCode(org.codelibs.fess.crawler.Constants.NOT_MODIFIED_STATUS);
                    processResponse(urlQueue, responseData);