// Default constructor
    }

    /**
     * Parses a robots.txt file from the given input stream using UTF-8 encoding.
     * @param stream the input stream to parse
     * @return the parsed RobotsTxt object, or null if disabled
     */
    public RobotsTxt parse(final InputStream stream) {
        return parse(stream, Constants.UTF_8);
    }

    /**

            spfactory.setFeature(Constants.FEATURE_EXTERNAL_PARAMETER_ENTITIES, false);
            // create a sax parser
            final SAXParser parser = spfactory.newSAXParser();
            try {
                parser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, StringUtil.EMPTY);
                parser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, StringUtil.EMPTY);
            } catch (final Exception e) {

            ReadingConverter readingConverter, Normalizer normalizer, SuggestAnalyzer analyzer, String[] langs);

    /**
     * Parses the given query log and returns a list of suggest items.
     *
     * @param queryLog the query log to parse
     * @param fields the fields to extract from the query log
     * @param tagFieldNames the names of the fields to use as tags

        final DOMParser parser = getDomParser();
        try (final Reader reader = new StringReader(content)) {
            parser.parse(new InputSource(reader));
        } catch (final Exception e) {
            logger.warn("Failed to parse the content.", e);
            return new ExtractData(extractString(content));
        }

        final Document document = parser.getDocument();
        try {

            SynonymMap.Builder parser = null;

            if ("wordnet".equalsIgnoreCase(settings.get("format"))) {
                parser = new WordnetSynonymParser(true, expand, analyzer);
                ((WordnetSynonymParser) parser).parse(rulesReader);
            } else {
                parser = new SolrSynonymParser(true, expand, analyzer);
                ((SolrSynonymParser) parser).parse(rulesReader);
            }

        try {
            sitemapsHelper.parse(in);
            fail();
        } catch (final CrawlingAccessException e) {
            // NOP
        }
    }

    public void test_parseXmlSitemaps_invalid2() {
        final byte[] bytes = "test".getBytes();
        final InputStream in = new ByteArrayInputStream(bytes);

        try {
            sitemapsHelper.parse(in);
            fail();

import jakarta.annotation.Resource;

/**
 * A response processor implementation that handles sitemaps.
 * It parses the response body as a SitemapSet, extracts URLs from the sitemaps,
 * and adds them as child URLs to be crawled.
 *
 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.
 * It then iterates through the sitemaps in the SitemapSet, extracts the URL

                                .collect(Collectors.toList());
                    } catch (final Exception e) {
                        logger.warn("Failed to parse passwords for " + url, e);
                        list = Collections.emptyList();
                    }
                    configPasswordMap.put(value, list);
                }

        nBestCost = settings.getAsInt(NBEST_COST, -1);
        nBestExamples = settings.get(NBEST_EXAMPLES);
    }

    private static String parse(String rule, Set<String> dup) {
        String[] values = CSVUtil.parse(rule);
        if (dup.add(values[0]) == false) {
            throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary.");
        }
        return rule;

    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        for (String userAgent : new String[] { "FessCrawler", "FessCrawler/1.0", "Mozilla FessCrawler" }) {