}
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }

    public void test_noFollowGetterSetter() {
        // Test noFollow getter/setter
        ResponseData data = new ResponseData();

        // Default should be false
        assertFalse(data.isNoFollow());

        data.setNoFollow(true);
        assertTrue(data.isNoFollow());

        data.setNoFollow(false);
        assertFalse(data.isNoFollow());
    }

    public void test_responseBodyBytes() {

crawler.document.html.canonical.xpath=//LINK[@rel='canonical'][1]/@href
# HTML tags to prune (remove) during document processing.
crawler.document.html.pruned.tags=noscript,script,style,header,footer,aside,nav,a[rel=nofollow]
# Maximum length of digest extracted from HTML documents.
crawler.document.html.max.digest.length=120
# Default language for HTML documents.
crawler.document.html.default.lang=

    String CRAWLER_DOCUMENT_HTML_CANONICAL_XPATH = "crawler.document.html.canonical.xpath";

    /** The key of the configuration. e.g. noscript,script,style,header,footer,aside,nav,a[rel=nofollow] */
    String CRAWLER_DOCUMENT_HTML_PRUNED_TAGS = "crawler.document.html.pruned.tags";

    /** The key of the configuration. e.g. 120 */

        assertEquals("text/html", responseData.getMimeType());
        assertEquals("UTF-8", responseData.getCharSet());
        assertEquals(1024L, responseData.getContentLength());
        assertFalse(responseData.isNoFollow());
    }

    public void test_processWithResponseDataContainingBody() {
        // Test process with ResponseData containing body
        NullResponseProcessor processor = new NullResponseProcessor();

    $ mvn dbflute:download # (one time command)
    $ mvn dbflute:freegen
    $ mvn license:format

### Integration Tests

Integration tests require a running Fess server with OpenSearch. Follow these steps:

#### 1. Build Fess

    $ mvn antrun:run  # Download OpenSearch plugins (if not done)
    $ mvn package     # Build the package

#### 2. Start Fess Server

    protected Map<String, String> filePathMap = new HashMap<>();

    /** The name of this thumbnail generator. */
    protected String name;

    /** Maximum number of redirects to follow. */
    protected int maxRedirectCount = 10;

    /** Availability status of this generator. */
    protected Boolean available = null;

    /**
     * Registers this thumbnail generator with the thumbnail manager.

4. **Edge Cases**: Null, empty, and boundary conditions are covered
5. **Resources**: Test resources are properly managed and cleaned up

## Contributing

When adding new tests:
1. Follow the existing naming convention: `test_<method>_<scenario>_<expectedResult>`
2. Add clear JavaDoc comments explaining the test purpose
3. Ensure tests are independent and can run in any order

    /** Maximum size of the delete URL cache before batch deletion is triggered. */
    protected int maxDeleteDocumentCacheSize;

    /** Maximum number of redirects to follow when processing URLs. */
    protected int maxRedirectCount;

    /** Executor service for concurrent processing of file operations. */
    private final ExecutorService executor;

        resultData.setTransformerName(getName());

        try {
            // data
            storeData(responseData, resultData);

            if (isHtml(responseData) && !responseData.isNoFollow()) {
                // urls
                storeChildUrls(responseData, resultData);
            }
        } finally {
            xpathAPI.remove();
        }