throw new CrawlingAccessException("Could not process " + robotTxtUrl + ". ", e);
        } finally {
            EntityUtils.consumeQuietly(httpEntity);
        }
    }

    /**
     * Converts a robots.txt pattern to a regular expression.
     *
     * @param path The robots.txt pattern to convert
     * @return The equivalent regular expression
     */

                .map(s -> s.split(":"))
                .filter(v -> v.length == 2)
                .map(v -> new Pair<String, String>(v[0].trim(), v[1].trim()));
    }

    /**
     * Converts and processes child URLs using path mapping and URL conversion rules.
     *
     * @param urlList the list of request data containing URLs to convert
     * @return the converted list of request data
     */

            this.textStart = textStart == null ? StringUtil.EMPTY : textStart;
            this.textEnd = textEnd;
            this.suffix = suffix;
        }

        /**
         * Converts this text fragment to a URL string parameter.
         *
         * @return the URL-encoded text fragment parameter
         */
        public String toURLString() {

import org.codehaus.plexus.util.xml.Xpp3Dom;
import org.eclipse.aether.graph.DependencyFilter;
import org.eclipse.aether.repository.RemoteRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * The concern of the project is provide runtime values based on the model.
 * <p>
 * The values in the model remain untouched but during the process of building a project notions like inheritance and

 * is similar policy for all of the 1.1 million Unicode code points. Note that some code points such
 * as "\ud83c\udf69" are not mapped and cannot be used in a hostname.
 *
 * [Punycode](http://ietf.org/rfc/rfc3492.txt) converts a Unicode string to an ASCII string to make
 * international domain names work everywhere. For example, "σ" encodes as "xn--4xa". The encoded