* Checks if the given URL uses a valid web protocol.
     *
     * @param url the URL to validate
     * @return true if the URL starts with a supported web protocol, false otherwise
     */
    public boolean isValidWebProtocol(final String url) {
        return stream(webProtocols).get(stream -> stream.anyMatch(s -> url.startsWith(s)));
    }

    /**
     * Checks if the given URL uses a valid file protocol.
     *

            // set urls
            split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
                if (!urlValue.startsWith("#") && protocolHelper.isValidWebProtocol(urlValue)) {
                    final String u = duplicateHostHelper.convert(urlValue);
                    crawler.addUrl(u);
                    if (logger.isInfoEnabled()) {