params.put("Offset", toNumberString(e.getQueryOffset()));
            params.put("Page Size", toNumberString(e.getQueryPageSize()));
            params.put("Client IP", e.getClientIp());
            params.put("Referer", e.getReferer());
            params.put("Languages", e.getLanguages());
            params.put("Virtual Host", e.getVirtualHost());

            long counter = 0;
            final Deque<String> urlQueue = new LinkedList<>();
            final Set<String> processedUrls = new HashSet<>();
            urlQueue.offer(url);
            while (!urlQueue.isEmpty() && (maxAccessCount < 0 || counter < maxAccessCount)) {
                counter++;
                final Map<String, Object> localDataMap =

    @Resource
    protected CrawlingInfoService crawlingInfoService;

    public static void addError(final String msg) {
        if (StringUtil.isNotBlank(msg)) {
            errors.offer(msg);
        }
    }

    public static class Options {

        @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
        public String sessionId;

        this.queryTime = value;
    }

    public String getReferer() {
        checkSpecifiedProperty("referer");
        return convertEmptyToNull(referer);
    }

    public void setReferer(String value) {
        registerModifiedProperty("referer");
        this.referer = value;
    }

    public LocalDateTime getRequestedAt() {
        checkSpecifiedProperty("requestedAt");

            false, "Long", 0, 0, null, null, false, null, null, null, null, null, false);
    protected final ColumnInfo _columnReferer = cci("referer", "referer", null, null, String.class, "referer", null, false, false, false,
            "keyword", 0, 0, null, null, false, null, null, null, null, null, false);

            result.setQueryPageSize(DfTypeUtil.toInteger(source.get("queryPageSize")));
            result.setQueryTime(DfTypeUtil.toLong(source.get("queryTime")));
            result.setReferer(DfTypeUtil.toString(source.get("referer")));
            result.setRequestedAt(toLocalDateTime(source.get("requestedAt")));
            result.setResponseTime(DfTypeUtil.toLong(source.get("responseTime")));
            result.setRoles(toStringArray(source.get("roles")));

            }
        } catch (final Exception e) {
            logger.warn("Failed to create thumbnail for {}", entity, e);
        }
    }

    public boolean offer(final Map<String, Object> docMap) {
        for (final ThumbnailGenerator generator : generatorList) {
            if (generator.isTarget(docMap)) {
                final String path = getImageFilename(docMap);

            searchLog.setClientIp(StringUtils.abbreviate(ComponentUtil.getViewHelper().getClientIp(req), 100));
            searchLog.setReferer(StringUtils.abbreviate(req.getHeader("referer"), 1000));
            searchLog.setUserAgent(StringUtils.abbreviate(req.getHeader("user-agent"), 255));
            final Object accessType = req.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);

                appendJson("query-page-size", entity.getQueryPageSize(), buf).append(',');
                appendJson("query-offset", entity.getQueryOffset(), buf).append(',');
                appendJson("referer", entity.getReferer(), buf).append(',');
                appendJson("languages", entity.getLanguages(), buf).append(',');
                appendJson("roles", entity.getRoles(), buf).append(',');

                final String thumbnailField = fessConfig.getIndexFieldThumbnail();
                docList.stream().forEach(doc -> {
                    if (!thumbnailManager.offer(doc)) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("Removing {}={} from doc[{}]", thumbnailField, doc.get(thumbnailField),