*/
    protected void storeChildUrls(final CrawlerContext crawlerContext, final Set<RequestData> childUrlList, final String url,
            final int depth, final String encoding) {
        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()

     * directory.
     */
    protected String path;

    /**
     * A string to replace ?.
     */
    protected String questionStr = "_QUEST_";

    /**
     * A string to replace :.
     */
    protected String colonStr = "_CLN_";

    /**
     * A string to replace ;.
     */
    protected String semicolonStr = "_SCLN_";

    /**
     * A string to replace &.
     */

     * Used for storing files in cloud storage services like S3.
     */
    @Size(max = 1000)
    public String storageEndpoint;

    /**
     * Access key for cloud storage authentication.
     * Used to authenticate with cloud storage services.
     */
    @Size(max = 1000)
    public String storageAccessKey;

    /**

     * and form inputs for the given parameters.
     *
     * @param paramMap the parameter map to process
     * @param queryKey the key for storing query string parameters
     * @param formKey the key for storing form input parameters
     */
    protected void buildInitParamMap(final Map<String, String> paramMap, final String queryKey, final String formKey) {

        logger.info("Creating new suggest index.");
        suggestHelper.suggester().createNextIndex();

        logger.info("Storing all bad words.");
        suggestHelper.storeAllBadWords(true);

        logger.info("Storing all elevate words.");
        suggestHelper.storeAllElevateWords(true);

        final AtomicInteger exitCode = new AtomicInteger(0);

            final StringBuilder sb = new StringBuilder(100);
            final List<String> fields = new ArrayList<>();
            final List<String> tags = new ArrayList<>();
            final List<String> roles = new ArrayList<>();

            for (final Pair<String, String> searchFieldLog : searchLog.getSearchFieldLogList()) {
                final String name = searchFieldLog.getFirst();
                if (contentFieldNameSet.contains(name)) {

            if (languages != null) {
                searchLog.setLanguages(StringUtils.join((String[]) languages, ","));
            } else {
                searchLog.setLanguages(StringUtil.EMPTY);
            }

            @SuppressWarnings("unchecked")
            final Map<String, List<String>> fieldLogMap = (Map<String, List<String>>) req.getAttribute(Constants.FIELD_LOGS);
            if (fieldLogMap != null) {

    public Object get(final Object key) {
        return getProperties().get(key);
    }

    @Override
    public String getProperty(final String key, final String defaultValue) {
        return getProperties().getProperty(key, defaultValue);
    }

    @Override
    public String getProperty(final String key) {
        return getProperties().getProperty(key);
    }

    @Override
    public int hashCode() {

    protected long cacheExpireAfterWrite = 10 * 60 * 1000L;

    /** Cache for storing statistics objects keyed by crawler object identifiers. */
    protected LoadingCache<String, StatsObject> statsCache;

    /**
     * Initializes the crawler statistics helper.
     * Sets up the statistics logger and creates the cache for storing
     * statistics objects with the configured size and expiration settings.
     */

     */
    default void putResultDataWithTemplate(final Map<String, Object> dataMap, final String key, final Object value, final String template,
            final String scriptType) {
        Object target = value;
        if (template != null) {
            final Map<String, Object> contextMap = new HashMap<>();
            contextMap.put("doc", dataMap);
            final Map<String, Object> paramMap = new HashMap<>(dataMap.size() + 2);