* @return the host name, or empty string if URL is blank, or unknown hostname if parsing fails
     */
    default String getHost(final String u) {
        if (StringUtil.isBlank(u)) {
            return StringUtil.EMPTY; // empty
        }

        String url = u;
        final String originalUrl = url;

        int idx = url.indexOf("://");
        if (idx >= 0) {
            url = url.substring(idx + 3);

            }
            if (values.length > 4 && "all".equals(values[4])) {
                return FormatType.SCROLL;
            }
            return FormatType.SEARCH;
        }
        if ("labels".equals(type)) {
            return FormatType.LABEL;
        }
        if ("popular-words".equals(type)) {
            return FormatType.POPULARWORD;
        }
        if ("favorites".equals(type)) {

    }

    /**
     * Checks if access to a given path is allowed for a specific user agent according to robots.txt rules.
     *
     * @param path The path to check for access permission
     * @param userAgent The user agent string to check against robots.txt directives
     * @return true if access is allowed, false if access is disallowed by robots.txt rules.
     *         Returns true if no matching directive is found for the user agent.

     * Validates the document ID, logs click events if enabled, and redirects
     * to the target URL or serves file content directly if configured.
     *
     * @param form the go form containing document ID and tracking parameters
     * @return action response for redirection or content streaming
     * @throws IOException if an I/O error occurs during content retrieval
     */
    @Execute

        String format = "0.#";
        if (value < 1024) {
            format = "0";
        } else if (value < 1024L * 1024L) {
            target /= 1024;
            unit = "K";
        } else if (value < 1024L * 1024L * 1024L) {
            target /= 1024;
            target /= 1024;
            unit = "M";
        } else if (value < 1024L * 1024L * 1024L * 1024L) {
            target /= 1024;

                if (cachedPathMappingList == null) {
                    init();
                }
            }
        }
        String result = text;
        for (final PathMapping pathMapping : cachedPathMappingList) {
            if (matchUserAgent(pathMapping)) {
                String replacement = pathMapping.getReplacement();
                if (replacement == null) {

            Object value = e.getValue();
            if (value == null) {
                value = StringUtil.EMPTY;
            }
            if (value instanceof String || value == null) {
                return new Pair<>(key, value);
            }
            if (arrayFieldSet.contains(key)) {
                if (value instanceof String[]) {

     * @throws CrawlingAccessException if the request fails
     */
    protected ResponseData processRequest(final String uri, final boolean includeContent) {
        if (isInit.compareAndSet(false, true)) {
            init();
        }

        // start
        AccessTimeoutTarget accessTimeoutTarget = null;
        TimeoutTask accessTimeoutTask = null;
        if (accessTimeout != null) {

        final JobExecutor jobExecutor = ComponentUtil.getJobExecutor(scriptType);
        if (jobExecutor == null) {
            throw new ScheduledJobException("No jobExecutor: " + scriptType);
        }

        if (!jobManager.findJobByUniqueOf(LaJobUnique.of(id)).isPresent()) {
            if (logger.isDebugEnabled()) {
                logger.debug("Job is running: id={}", id);
            }
            return;

                }

                if (file.canRead()) {
                    final MimeTypeHelper mimeTypeHelper = crawlerContainer.getComponent("mimeTypeHelper");
                    if (includeContent) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("Parsing SmbFile Content: {}", filePath);
                        }