*/
        public SitemapSet getSitemapSet() {
            return sitemapSet;
        }

    }

    /**
     * Sets the preload size for checking file format.
     * @param preloadSize the preload size in bytes
     */
    public void setPreloadSize(final int preloadSize) {
        this.preloadSize = preloadSize;
    }

                }
                responseData.addMetaData(SMB_ALLOWED_SID_ENTRIES, sidAllowSet.toArray(new SID[sidAllowSet.size()]));
                responseData.addMetaData(SMB_DENIED_SID_ENTRIES, sidDenySet.toArray(new SID[sidDenySet.size()]));
            }
        } catch (final IOException e) {
            throw new CrawlingAccessException("Could not access " + file.getPath(), e);
        }

    private static boolean removeLastDuplication(final StringBuilder buf, final int size, final boolean isSpace,
            final Set<String> termCache) {
        final String target = rightString(buf, size);
        if (!termCache.contains(target)) {
            termCache.add(target);
            return isSpace;
        }
        buf.setLength(buf.length() - size);
        if (!isSpace && !isLastSpaceChar(buf)) {

        List<String> list = extractor.parseCommand(cmd, params);
        assertEquals(0, list.size());

        cmd = "test.sh";
        params.clear();
        list = extractor.parseCommand(cmd, params);
        assertEquals(1, list.size());
        assertEquals("test.sh", list.get(0));

        cmd = "test.sh \"test1 ' test2\"";
        params.clear();

            FileUtil.deleteInBackground(tempFile);
        }

        return new ExtractData(buf.toString().trim());
    }

    /**
     * Sets the maximum content size for extraction.
     *
     * @param maxContentSize the maximum content size to set (-1 for no limit)
     */
    public void setMaxContentSize(final long maxContentSize) {
        this.maxContentSize = maxContentSize;
    }

        final List<String> wordList = Analysis.parseWordList(environment, settings, "stopwords", s -> s);
        if (wordList != null) {
            stopwords = wordList.toArray(new String[wordList.size()]);
        } else {
            stopwords = new String[0];
        }

        ignoreCase = settings.getAsBoolean("ignore_case", Boolean.FALSE).booleanValue();
        if (ignoreCase) {

     * @param description The description of the exception.
     */
    public ChildUrlsException(final Set<RequestData> childUrlList, final String description) {
        super("Threw child urls(" + childUrlList.size() + "). " + description, false, false);
        this.childUrlList = childUrlList;
    }

    /**
     * Returns the list of child URLs.
     * @return The list of child URLs.
     */

            CrawlingParameterUtil.setCrawlerContext(crawlerContext);
            httpClient.init();
            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());
            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));

        return new SuggestItem(new String[] { this.getElevateWord() }, readingArray, fields.toArray(new String[fields.size()]), 1, 0,
                this.getBoost(), this.getTags().toArray(new String[this.getTags().size()]),
                this.getRoles().toArray(new String[this.getRoles().size()]), null, SuggestItem.Kind.USER);
    }

        item.readings = readings.toArray(new String[readings.size()][]);
        item.fields = fields.toArray(new String[fields.size()]);
        item.docFreq = docFreq;
        item.queryFreq = queryFreq;
        item.userBoost = userBoost;
        item.tags = tags.toArray(new String[tags.size()]);
        item.roles = roles.toArray(new String[roles.size()]);
        item.languages = languages.toArray(new String[languages.size()]);