log(logHelper, LogType.PROCESS_CHILD_URLS_BY_EXCEPTION, crawlerContext, urlQueue, childUrlSet);
                            // add an url
                            storeChildUrls(childUrlSet, urlQueue.getUrl(), urlQueue.getDepth() == null ? 1 : urlQueue.getDepth() + 1);
                        } catch (final Exception e1) {

        try {
            // data
            storeData(responseData, resultData);

            if (isHtml(responseData) && !responseData.isNoFollow()) {
                // urls
                storeChildUrls(responseData, resultData);
            }
        } finally {
            xpathAPI.remove();
        }

        final Object redirectUrlObj = responseData.getMetaDataMap().get(LOCATION_HEADER);

                }
                if (noindex) {
                    logger.info("META(robots=noindex): {}", responseData.getUrl());
                    storeChildUrls(responseData, resultData);
                    throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
                }
                if (nofollow) {