if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            return new ExtractData(new String(InputStreamUtil.getBytes(in), getEncoding()));
        } catch (final Exception e) {
            throw new ExtractException(e);
        }
    }

    /**
     * Returns the encoding used for text extraction.
     * @return the encoding
     */

     */
    public MimeTypeHelperImpl() {
        try {
            mimeTypes = MimeTypesFactory.create(MIME_TYPES_RESOURCE_NAME);
        } catch (final Exception e) {
            throw new CrawlerSystemException("Could not initialize MimeTypeHelper.", e);
        }
    }

    @Override
    public String getContentType(final InputStream is, final String filename) {

                try (InputStream is = getContentInputStream(out)) {
                    return extractor.getText(is, params);
                }
            }
        } catch (final CrawlingAccessException e) {
            throw e;
        } catch (final Exception e) {
            throw new ExtractException("Failed to extract data.", e);
        } finally {
            if (dfos != null && !dfos.isInMemory()) {

     * @return The encoding pattern.
     */
    @Override
    protected Pattern getEncodingPattern() {
        return xmlEncodingPattern;
    }

    /**
     * Returns the precompiled {@link Pattern} used to match XML tags within the content.
     * This pattern is utilized by the extractor to identify and process XML elements.
     *
     * @return the {@link Pattern} instance for XML tag matching
     */
    @Override

            try (Reader reader = IOUtils.getDecodingReader(new FileInputStream(suffixWordsFile), StandardCharsets.UTF_8)) {
                suffixWords = WordlistLoader.getWordSet(reader);
            } catch (final IOException e) {
                throw new IllegalArgumentException("Could not load " + suffixWordsFile.getAbsolutePath(), e);
            }
        } else {
            suffixWords = new CharArraySet(0, false);

     * Gets the first rule that matches the given response data.
     * @param responseData the response data to match against
     * @return the first matching rule, or null if no rule matches
     */
    @Override
    public Rule getRule(final ResponseData responseData) {
        for (final Rule rule : ruleList) {
            if (rule.match(responseData)) {
                return rule;
            }
        }
        return null;
    }

        assertTrue(sitemapsRule.match(responseData));
        InputStream is = responseData.getResponseBody();
        assertTrue(is instanceof InputStream);
        CloseableUtil.closeQuietly(responseData);
    }

    private void assertMatchFalse(ResponseData responseData) {
        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));
        assertFalse(sitemapsRule.match(responseData));

                } else {
                    String content;
                    try {
                        content = new String(InputStreamUtil.getBytes(entity.getContent()), Constants.UTF_8_CHARSET);
                    } catch (final IOException e) {
                        content = e.getMessage();
                    }
                    logger.warn("Failed to access to {}. The http status is {}.\n{}", tokenUrl, httpStatusCode, content);

                CopyUtil.copy(is, os);
            } catch (final IOException e) {
                throw new CrawlerSystemException("Could not store " + file.getAbsolutePath(), e);
            }
        }
        try {
            resultData.setData(path.getBytes(charsetName));
        } catch (final UnsupportedEncodingException e) {
            if (logger.isInfoEnabled()) {

 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    /** Map of user agent patterns to their corresponding directives. */