* Gets the first rule that matches the given response data.
     * @param responseData the response data to match against
     * @return the first matching rule, or null if no rule matches
     */
    @Override
    public Rule getRule(final ResponseData responseData) {
        for (final Rule rule : ruleList) {
            if (rule.match(responseData)) {
                return rule;
            }
        }
        return null;
    }

        assertTrue(sitemapsRule.match(responseData));
        InputStream is = responseData.getResponseBody();
        assertTrue(is instanceof InputStream);
        CloseableUtil.closeQuietly(responseData);
    }

    private void assertMatchFalse(ResponseData responseData) {
        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));
        assertFalse(sitemapsRule.match(responseData));

                } else {
                    String content;
                    try {
                        content = new String(InputStreamUtil.getBytes(entity.getContent()), Constants.UTF_8_CHARSET);
                    } catch (final IOException e) {
                        content = e.getMessage();
                    }
                    logger.warn("Failed to access to {}. The http status is {}.\n{}", tokenUrl, httpStatusCode, content);

     */
    void init(String sessionId);

    /**
     * Check if a given url is a target.
     *
     * @param url URL
     * @return true if url is matched
     */
    boolean match(String url);

    /**
     * Add an url pattern as a target.
     *
     * @param urlPattern Regular expression that is crawled
     */
    void addInclude(String urlPattern);

    /**

                CopyUtil.copy(is, os);
            } catch (final IOException e) {
                throw new CrawlerSystemException("Could not store " + file.getAbsolutePath(), e);
            }
        }
        try {
            resultData.setData(path.getBytes(charsetName));
        } catch (final UnsupportedEncodingException e) {
            if (logger.isInfoEnabled()) {

 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    /** Map of user agent patterns to their corresponding directives. */

        logger.info(content);
        assertTrue(content.contains("テスト"));
    }

    public void test_getText_null() {
        try {
            msWordExtractor.getText(null, null);
            fail();
        } catch (final CrawlerSystemException e) {
            // NOP
        }
    }

            @SuppressWarnings("resource")
            final PublisherTextExtractor publisherTextExtractor = new PublisherTextExtractor(in);
            return new ExtractData(publisherTextExtractor.getText());
        } catch (final IOException e) {
            throw new ExtractException(e);
        }
    }

                break;
            default:
                // No operation
                break;
            }
        } catch (final CrawlerSystemException e) {
            if (!ignoreException) {
                throw e;
            }
        } catch (final Exception e) {
            if (!ignoreException) {
                throw new CrawlerSystemException("Could not stop a process.", e);
            }

                        break;
                    }
                    lastTime.wait(delayTime);
                }
            }
        } catch (final InterruptedException e) {
            throw new InterruptedRuntimeException(e);
        } catch (final Exception e) {
            throw new CrawlerSystemException(e);
        }
    }