/** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

- **Testing**: JUnit 4, UTFlute, Mockito 5.7.0
- **Storage**: In-memory (default), OpenSearch (optional)

### Protocols

- **HTTP/HTTPS**: Full crawling, cookies, auth, robots.txt
- **File**: Local/network file systems
- **FTP**: With authentication
- **SMB/CIFS**: Windows shares (SMB1/SMB2+)
- **Storage**: MinIO/S3-compatible

### Content Formats

        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

            httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

        }
        if (logger.isDebugEnabled()) {
            logger.debug("Loaded ScriptEngine: {}", name);
        }
        scriptEngineMap.put(name.toLowerCase(Locale.ROOT), scriptEngine);
        scriptEngineMap.put(scriptEngine.getClass().getSimpleName().toLowerCase(Locale.ROOT), scriptEngine);
    }

    /**
     * Gets a script engine.
     * @param name The name of the script engine.
     * @return The script engine.
     */

    /** The key of the configuration. e.g. [Test] */
    String MAIL_SUBJECT_TEST_PREFIX = "mail.subject.test.prefix";

    /** The key of the configuration. e.g. root@localhost */
    String MAIL_RETURN_PATH = "mail.return.path";

    /**
     * Get the value of property as {@link String}.
     * @param propertyKey The key of the property. (NotNull)

     * Retrieves the current user's locale from the request manager.
     *
     * @return the user's locale, or Locale.ROOT if not available
     */
    private static Locale getUserLocale() {
        final Locale locale = ComponentUtil.getRequestManager().getUserLocale();
        if (locale == null) {
            return Locale.ROOT;
        }
        return locale;
    }

    /**

            final Map<String, List<String>> map = new HashMap<>();
            headers.entrySet().stream().filter(e -> e.getKey() != null).forEach(
                    e -> map.put(e.getKey().toLowerCase(Locale.ROOT), e.getValue() == null ? List.of() : List.copyOf(e.getValue())));
            this.headers = Map.copyOf(map);
        }
    }

    /**
     * Gets the headers of the response.
     *

                }
                return language.toLowerCase(Locale.ROOT);
            }
            return null;
        }).toArray(n -> new String[n]));
    }

    default Locale getQueryLocaleFromName(final String name) {
        if (name == null) {
            return Locale.ROOT;
        }