---

## Architecture

### Module Structure

```
fess-crawler-parent/
├── fess-crawler/              # Core framework
├── fess-crawler-lasta/        # LastaFlute DI integration
└── fess-crawler-opensearch/   # OpenSearch backend
```

### Key Design Patterns

**Factory**: `CrawlerClientFactory`, `ExtractorFactory` - protocol/format-specific component selection

mvn test

# Run single unit test
mvn test -Dtest=SearchEngineClientTest

# Run integration tests (*Tests.java in src/test/java/org/codelibs/fess/it/)
# Requires Fess server and OpenSearch running
mvn test -P integrationTests -Dtest.fess.url="http://localhost:8080" -Dtest.search_engine.url="http://localhost:9201"

# Run single integration test

    $ mvn jdeb:jdeb # .deb package

### Generate Source Code

    $ mvn dbflute:download # (one time command)
    $ mvn dbflute:freegen
    $ mvn license:format

### Integration Tests

Integration tests require a running Fess server with OpenSearch. Follow these steps:

#### 1. Build Fess

    $ mvn antrun:run  # Download OpenSearch plugins (if not done)
    $ mvn package     # Build the package

import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.opensearch.action.bulk.BulkRequestBuilder;
import org.opensearch.action.index.IndexAction;
import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.search.sort.SortBuilders;
import org.opensearch.search.sort.SortOrder;

import org.apache.pdfbox.pdmodel.common.filespecification.PDEmbeddedFile;
import org.apache.pdfbox.pdmodel.common.filespecification.PDFileSpecification;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationFileAttachment;
import org.apache.pdfbox.text.PDFTextStripper;
import org.codelibs.core.lang.ThreadUtil;
import org.codelibs.fess.crawler.entity.ExtractData;

    - name: Build with Maven
      run: mvn -B source:jar javadoc:jar package --file pom.xml
    - name: Run Fess
      run: bash src/test/resources/before_script.sh
    - name: Run Integration Test

 * modification checking, and integration with the Fess search engine backend.
 *
 * <p>Key features include:</p>
 * <ul>
 * <li>Incremental crawling support with last-modified timestamp checking</li>
 * <li>Document expiration handling</li>
 * <li>Child URL extraction and queueing</li>
 * <li>Integration with Fess configuration and permission systems</li>

     * Controls the level of detail in system log messages.
     */
    @Size(max = 10)
    public String logLevel;

    /**
     * Storage service endpoint URL for cloud storage integration.
     * Used for storing files in cloud storage services like S3.
     */
    @Size(max = 1000)
    public String storageEndpoint;

    /**
     * Access key for cloud storage authentication.

     */
    public SuggestCreator() {
        // do nothing
    }

    private static final Logger logger = LogManager.getLogger(SuggestCreator.class);

    /** The search engine client for interacting with OpenSearch. */
    @Resource
    public SearchEngineClient searchEngineClient;

    /**
     * A nested class for parsing command-line options.
     */
    protected static class Options {

import groovy.lang.GroovyShell;

/**
 * Groovy script engine implementation that extends AbstractScriptEngine.
 * This class provides support for executing Groovy scripts with parameter binding
 * and DI container integration.
 *
 * <p>Thread Safety: This class is thread-safe. Each evaluate() call creates
 * a new GroovyShell instance to ensure thread isolation.</p>
 *
 * <p>Resource Management: GroovyClassLoader instances are properly managed