*/
    private static int process(final Options options) {
        final Crawler crawler = ComponentUtil.getComponent(Crawler.class);

        if (StringUtil.isBlank(options.sessionId)) {
            // use a default session id
            final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
            options.sessionId = sdf.format(new Date());
        } else {

                searchResult = searchByUrl(intentResult.getDocumentUrl());
            } else {
                final String query = StringUtil.isBlank(intentResult.getQuery()) ? userMessage : intentResult.getQuery();
                searchResult = searchWithQueryAndMetadata(query, safeFields, safeExtraQueries);

                // Fallback: regenerate query if no results

                    user.getId() != null ? user.getId() : "new");
        }

        try {
            if (StringUtil.isBlank(user.getSurname())) {
                user.setSurname(user.getName());
            }

            ComponentUtil.getAuthenticationManager().insert(user);

            userBhv.insertOrUpdate(user, op -> {

                return;
            }
            final String sessionId = initParamMap.getAsString(Constants.SESSION_ID);
            if (StringUtil.isBlank(sessionId)) {
                logger.warn("[{}] Cannot delete stale documents: sessionId is not set.", dataConfig.getName());
                return;
            }

     *
     * @param text the text to strip HTML tags from
     * @return the text without HTML tags
     */
    protected String stripHtmlTags(final String text) {
        if (StringUtil.isBlank(text)) {
            return text;
        }
        return text.replaceAll("<[^>]+>", "");
    }

    /**
     * Sanitizes document content by escaping delimiter-like sequences

        final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
        thumbnailQueueBhv.selectList(cb -> {
            if (StringUtil.isBlank(fessConfig.getSchedulerTargetName())) {
                cb.query().setTarget_Equal(Constants.DEFAULT_JOB_TARGET);
            } else {

C7BE,EAAgBF,EAAU,MAAVA,CAAmB,MACnCG,EAAcH,EAAU,QAAVA,CAAqB,QACnCI,EAAuB,EAAsB,OAAtB,CAAW,qBAGtCC,KACAA,KAAgC,CADhCA,CAEAR,KAA0B,OACxB3B,MAEAmC,KAAkCR,KAGlCQ,EAAiBC,IAAjBD,IChCN,eAAyC,OAEnCE,OAAMC,SAAND,CAAgBE,IAFmB,CAG9BC,EAAID,IAAJC,GAH8B,CAOhCA,EAAI1B,MAAJ0B,IAAkB,CAAlBA,ECLT,iBAAoD,IAE9CH,MAAMC,SAAND,CAAgBI,gBACXD,GAAIC,SAAJD,CAAc,kBAAOE,SAArB,CAAAF,KAIHG,GAAQJ,IAAU,kBAAOK,SAAjB,CAAAL,QACPC,GAAInI,OAAJmI,ICLT,iBAA4D,IACpDK,GAAiBC,aAEnBC,EAAUC,KAAVD,CAAgB,CAAhBA,CAAmBN,IAAqB,MAArBA,GAAnBM,WAEW...

     * @return a list of pairs containing client names and their corresponding compiled patterns
     */
    protected List<Pair<String, Pattern>> getClientRuleList(final String value) {
        if (StringUtil.isBlank(value)) {
            return Collections.emptyList();
        }
        return split(value, ",").get(stream -> stream.map(String::trim)//
                .map(s -> clientRuleCache.computeIfAbsent(s, t -> {

            crawler.setSessionId(sid);
            sessionIdList.add(sid);

            final String urlsStr = webConfig.getUrls();
            if (StringUtil.isBlank(urlsStr)) {
                logger.warn("[{}] No target urls. Skipped.", webConfig.getName());
                break;
            }

            // interval time
            final int intervalTime =

        return split(text, "\n")
                .get(stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).map(this::getFilterPath).filter(s -> {
                    if (StringUtil.isBlank(s)) {
                        return false;
                    }
                    if (Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))) {
                        return web;
                    }