assertTrue(exceptions.isEmpty());
        assertEquals(threadCount * objectsPerThread, createdComponents.size());

        // Check all components are unique (for prototype)
        for (int i = 0; i < createdComponents.size(); i++) {
            for (int j = i + 1; j < createdComponents.size(); j++) {
                assertNotSame(createdComponents.get(i), createdComponents.get(j));
            }
        }
    }

         */
        public SitemapSet getSitemapSet() {
            return sitemapSet;
        }

    }

    /**
     * Sets the preload size for checking file format.
     * @param preloadSize the preload size in bytes
     */
    public void setPreloadSize(final int preloadSize) {
        this.preloadSize = preloadSize;
    }

                }
                responseData.addMetaData(SMB_ALLOWED_SID_ENTRIES, sidAllowSet.toArray(new SID[sidAllowSet.size()]));
                responseData.addMetaData(SMB_DENIED_SID_ENTRIES, sidDenySet.toArray(new SID[sidDenySet.size()]));
            }
        } catch (final IOException e) {
            throw new CrawlingAccessException("Could not access " + file.getPath(), e);
        }

    }

    /**
     * Returns the preload size for charset detection.
     * @return The preload size for charset detection.
     */
    public int getPreloadSizeForCharset() {
        return preloadSizeForCharset;
    }

    /**
     * Sets the preload size for charset detection.
     * @param preloadSizeForCharset The preload size for charset detection to set.
     */

    private static boolean removeLastDuplication(final StringBuilder buf, final int size, final boolean isSpace,
            final Set<String> termCache) {
        final String target = rightString(buf, size);
        if (!termCache.contains(target)) {
            termCache.add(target);
            return isSpace;
        }
        buf.setLength(buf.length() - size);
        if (!isSpace && !isLastSpaceChar(buf)) {

        }

        @Override
        public void addRule(int index, Rule rule) {
            if (rule != null) {
                if (index < 0 || index > rules.size()) {
                    throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + rules.size());
                }
                rules.add(index, rule);
            }
        }

        @Override
        public boolean removeRule(Rule rule) {

    /** The property name for maximum cached content size. */
    public static final String MAX_CACHED_CONTENT_SIZE = "maxCachedContentSize";

    /** The crawler container. */
    @Resource
    protected CrawlerContainer crawlerContainer;

    /** The initialization parameter map. */
    protected Map<String, Object> initParamMap;

    /** The maximum cached content size in bytes. Default is 1MB. */

            CrawlingParameterUtil.setCrawlerContext(crawlerContext);
            httpClient.init();
            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());
            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));

        List<String> list = extractor.parseCommand(cmd, params);
        assertEquals(0, list.size());

        cmd = "test.sh";
        params.clear();
        list = extractor.parseCommand(cmd, params);
        assertEquals(1, list.size());
        assertEquals("test.sh", list.get(0));

        cmd = "test.sh \"test1 ' test2\"";
        params.clear();

    }

    /**
     * Gets the scroll size for search operations.
     *
     * @return The scroll size.
     */
    public int getScrollSize() {
        return scrollSize;
    }

    /**
     * Sets the scroll size for search operations.
     *
     * @param scrollSize The scroll size.
     */
    public void setScrollSize(final int scrollSize) {