* @param stream the input stream to parse
     * @return the parsed RobotsTxt object, or null if disabled
     */
    public RobotsTxt parse(final InputStream stream) {
        return parse(stream, Constants.UTF_8);
    }

    /**
     * Parses a robots.txt file from the given input stream using the specified character encoding.
     * @param stream the input stream to parse
     * @param charsetName the character encoding to use

     */
    public SitemapSet parse(final InputStream in) {
        return parse(in, true);
    }

    /**
     * Parses a sitemap from the given input stream.
     * @param in the input stream to parse
     * @param recursive whether to recursively parse compressed files
     * @return the parsed sitemap set
     */
    protected SitemapSet parse(final InputStream in, final boolean recursive) {

        } finally {
            FileUtil.deleteInBackground(tempFile);
        }
    }

    /**
     * Creates a parse context.
     *
     * @param parser A parser.
     * @param params A map of parameters.
     * @return a parse context.
     */
    protected ParseContext createParseContext(final Parser parser, final Map<String, String> params) {

            } catch (final Exception e) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Failed to set a property.", e);
                }
            }
            // parse a content
            parser.parse(is, handler);

            return handler.getDataMap();
        } catch (final Exception e) {
            throw new CrawlerSystemException("Could not create a data map from XML content.", e);
        }

 * It parses the response body as a SitemapSet, extracts URLs from the sitemaps,
 * and adds them as child URLs to be crawled.
 *
 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.
 * It then iterates through the sitemaps in the SitemapSet, extracts the URL
 * from each sitemap, and creates a new {@link RequestData} object for each URL.

            File file = null;
            try {
                file = new File(new URI(filePath));
            } catch (final URISyntaxException e) {
                logger.warn("Could not parse url: " + filePath, e);
            }

            if (file == null) {
                responseData.setHttpStatusCode(Constants.NOT_FOUND_STATUS_CODE);
                responseData.setCharSet(charset);

        final DOMParser parser = getDomParser();
        try (final Reader reader = new StringReader(content)) {
            parser.parse(new InputSource(reader));
        } catch (final Exception e) {
            logger.warn("Failed to parse the content.", e);
            return new ExtractData(extractString(content));
        }

        final Document document = parser.getDocument();
        try {

            if (responseData.getCharSet() != null) {
                is.setEncoding(responseData.getCharSet());
            }
            parser.parse(is);
        } catch (final Exception e) {
            throw new CrawlingAccessException("Could not parse " + responseData.getUrl(), e);
        }
        final Document document = parser.getDocument();

        final StringBuilder buf = new StringBuilder(1000);

                                .collect(Collectors.toList());
                    } catch (final Exception e) {
                        logger.warn("Failed to parse passwords for " + url, e);
                        list = Collections.emptyList();
                    }
                    configPasswordMap.put(value, list);
                }

                        }
                    }
                }
            } catch (final IOException e) {
                logger.warn("Failed to parse annotation.", e);
            }
        }
    }

    /**
     * Extracts text from an embedded file using the appropriate extractor.
     * @param filename the filename of the embedded file