*   <li>Processing the response using a {@link ResponseProcessor} associated with a {@link Rule}.</li>
 *   <li>Extracting child URLs and adding them to the queue using {@link #storeChildUrls(Set, String, int)}
 *       or {@link #storeChildUrl(String, String, float, int)}.</li>
 *   <li>Handling exceptions that may occur during the crawling process.</li>
 * </ol>
 *
 * <p>

     */
    protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) {
        if (childUrlSet != null) {
            // add an url
            try {
                storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                        urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);

                }
                if (noindex) {
                    logger.info("META(robots=noindex): {}", responseData.getUrl());
                    storeChildUrls(responseData, resultData);
                    throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
                }
                if (nofollow) {