}
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }

crawler.document.html.canonical.xpath=//LINK[@rel='canonical'][1]/@href
# HTML tags to prune (remove) during document processing.
crawler.document.html.pruned.tags=noscript,script,style,header,footer,aside,nav,a[rel=nofollow]
# Maximum length of digest extracted from HTML documents.
crawler.document.html.max.digest.length=120
# Default language for HTML documents.
crawler.document.html.default.lang=

    String CRAWLER_DOCUMENT_HTML_CANONICAL_XPATH = "crawler.document.html.canonical.xpath";

    /** The key of the configuration. e.g. noscript,script,style,header,footer,aside,nav,a[rel=nofollow] */
    String CRAWLER_DOCUMENT_HTML_PRUNED_TAGS = "crawler.document.html.pruned.tags";

    /** The key of the configuration. e.g. 120 */

    $ mvn dbflute:download # (one time command)
    $ mvn dbflute:freegen
    $ mvn license:format

### Integration Tests

Integration tests require a running Fess server with OpenSearch. Follow these steps:

#### 1. Build Fess

    $ mvn antrun:run  # Download OpenSearch plugins (if not done)
    $ mvn package     # Build the package

#### 2. Start Fess Server

    protected Map<String, String> filePathMap = new HashMap<>();

    /** The name of this thumbnail generator. */
    protected String name;

    /** Maximum number of redirects to follow. */
    protected int maxRedirectCount = 10;

    /** Availability status of this generator. */
    protected Boolean available = null;

    /**
     * Registers this thumbnail generator with the thumbnail manager.

    /** Maximum size of the delete URL cache before batch deletion is triggered. */
    protected int maxDeleteDocumentCacheSize;

    /** Maximum number of redirects to follow when processing URLs. */
    protected int maxRedirectCount;

    /** Executor service for concurrent processing of file operations. */
    private final ExecutorService executor;

        resultData.setTransformerName(getName());

        try {
            // data
            storeData(responseData, resultData);

            if (isHtml(responseData) && !responseData.isNoFollow()) {
                // urls
                storeChildUrls(responseData, resultData);
            }
        } finally {
            xpathAPI.remove();
        }

    // Process
}  // Temp files auto-deleted
```

---

## Best Practices for AI Assistants

### When Adding Features

1. Read existing code first (use symbol overview tools)
2. Follow existing patterns
3. Add tests
4. Handle resources properly (try-with-resources)
5. Consider thread safety
6. Update JavaDoc

### When Fixing Bugs

1. Write failing test first
2. Understand root cause