}

    /**
     * Gets the map of child URL extraction rules.
     *
     * @return the child URL rule map
     */
    public Map<String, String> getChildUrlRuleMap() {
        return childUrlRuleMap;
    }

    /**
     * Sets the map of child URL extraction rules.
     *
     * @param childUrlRuleMap the child URL rule map to set
     */

        return metaDataMap;
    }

    /**
     * Adds a child URL discovered from this response.
     *
     * @param url the child URL to add
     */
    public void addChildUrl(final RequestData url) {
        childUrlSet.add(url);
    }

    /**
     * Removes a child URL from this response.
     *
     * @param url the child URL to remove
     */

        urlQueueService.offerAll(crawlerContext.sessionId, childList);
    }

    /**
     * Stores a single child URL to the crawling queue.
     * @param childUrl The child URL to store.
     * @param parentUrl The parent URL.
     * @param weight The weight of the child URL.
     * @param depth The depth of the child URL.
     */

        this.data = data;
    }

    /**
     * Add a child URL.
     * @param url the request data to add to the child URL set
     */
    public void addUrl(final RequestData url) {
        childUrlSet.add(url);
    }

    /**
     * Add child URLs.
     * @param c the collection of request data to add to the child URL set
     */
    public void addAllUrl(final Collection<RequestData> c) {

                        SID.SID_FLAG_RESOLVE_SIDS);
                for (final SID child : children) {
                    if (!sidSet.contains(child)) {
                        processAllowedSIDs(file, child, sidSet);
                    }
                }
            } catch (final Exception e) {
                if (logger.isDebugEnabled()) {

/**
 * {@link ChildUrlsException} is thrown when child URLs are found during crawling.
 * It extends {@link CrawlerSystemException} and holds a set of {@link RequestData}
 * representing the child URLs that caused the exception.
 *
 */
public class ChildUrlsException extends CrawlerSystemException {

    private static final long serialVersionUID = 1L;

    /**
     * The list of child URLs.
     */

    /** Indicates processing a response. */
    PROCESS_RESPONSE,
    /** Indicates the crawling process has finished. */
    FINISHED_CRAWLING,
    /** Indicates processing child URLs due to an exception. */
    PROCESS_CHILD_URLS_BY_EXCEPTION,
    /** Indicates processing a child URL due to an exception. */
    PROCESS_CHILD_URL_BY_EXCEPTION,
    /** Indicates an access exception during crawling. */
    CRAWLING_ACCESS_EXCEPTION,

     * @return the response data containing metadata and optionally content
     * @throws CrawlingAccessException if an error occurs while accessing the resource
     * @throws ChildUrlsException if the URI represents a directory with child URLs
     */
    protected ResponseData getResponseData(final String uri, final boolean includeContent) {
        final ResponseData responseData = new ResponseData();

        reader.setScrollSize(1000);
        int count = 0;
        Set<String> valueSet = Collections.synchronizedSet(new HashSet<>());
        Map<String, Object> source;
        while ((source = reader.read()) != null) {
            assertTrue(source.get("field1").toString().startsWith("test"));
            valueSet.add(source.get("field1").toString());
            count++;
        }

        }
        return true;
    }

    /**
     * Stores child URLs found in the response data.
     *
     * @param crawlerContext the crawler context
     * @param childUrlList the set of child URLs
     * @param url the parent URL
     * @param depth the depth of the child URLs
     * @param encoding the encoding of the child URLs
     */