// Test multiple User-agents in sequence (Bot1, Bot2, Bot3 should share the same rules)
        assertFalse(robotsTxt.allows("/shared/", "Bot1"));
        assertFalse(robotsTxt.allows("/shared/", "Bot2"));
        assertFalse(robotsTxt.allows("/shared/", "Bot3"));

        // Test sitemaps - should parse valid sitemaps and ignore invalid ones

import jakarta.annotation.Resource;

/**
 * The {@link SmbClient} class is a crawler client implementation for accessing files and directories
 * on SMB (Server Message Block) shares. It extends {@link AbstractCrawlerClient} and utilizes the JCIFS library
 * to interact with SMB resources.
 *
 * <p>
 * This client supports authentication, content retrieval, and metadata extraction from SMB files.

import org.codelibs.jcifs.smb1.util.LogStream;

import jakarta.annotation.Resource;

/**
 * The {@link SmbClient} class is a crawler client implementation for accessing files and directories
 * on SMB (Server Message Block) shares using the SMB1 protocol. It extends {@link AbstractCrawlerClient} and utilizes the JCIFS library
 * to interact with SMB resources.
 *
 * <p>

	</modules>
	<build>
		<pluginManagement>
			<plugins>
				<plugin>
					<groupId>org.apache.maven.plugins</groupId>
					<artifactId>maven-shade-plugin</artifactId>
					<executions>
						<execution>
							<phase>package</phase>
							<goals>
								<goal>shade</goal>
							</goals>
							<configuration>
								<artifactSet>
									<includes>
										<include>org.dbflute:dbflute-runtime</include>

### Protocols

- **HTTP/HTTPS**: Full crawling, cookies, auth, robots.txt
- **File**: Local/network file systems
- **FTP**: With authentication
- **SMB/CIFS**: Windows shares (SMB1/SMB2+)
- **Storage**: MinIO/S3-compatible

### Content Formats

Office (Word, Excel, PowerPoint), PDF, Archives (ZIP, TAR, GZ), HTML, XML, JSON, Media (audio/video metadata), Images (EXIF/IPTC/XMP)

---

            { "aliases", "allocation", "count", "fielddata", "health", "indices", "master", "nodeattrs", "nodes", "pending_tasks",
                    "plugins", "recovery", "repositories", "thread_pool", "shards", "segments", "snapshots", "templates" };

    // ===================================================================================
    //                                                                           Attribute

        q.setUrl("http://example.com/page1");
        CrawlingParameterUtil.setUrlQueue(q);

        // First access - should not delay
        final long start1 = System.nanoTime();
        controller.delayBeforeProcessing();
        final long elapsed1 = (System.nanoTime() - start1) / 1000000;
        assertTrue("First access should not delay", elapsed1 < 50);

        // Second access to same host - should delay

            }
            if (path == null) {
                path = System.getenv("path");
            }
            final List<String> pathList = new ArrayList<>();
            pathList.add("/usr/share/fess/bin");
            if (path != null) {
                stream(path.split(File.pathSeparator)).of(stream -> stream.map(String::trim).forEach(s -> pathList.add(s)));
            }
            if (logger.isDebugEnabled()) {

		<groupId>org.codelibs.fess</groupId>
		<artifactId>fess-parent</artifactId>
		<version>15.4.0</version>
		<relativePath />
	</parent>
	<properties>
		<!-- DEB & RPM build -->
		<packaging.fess.home.dir>/usr/share/fess</packaging.fess.home.dir>
		<packaging.fess.app.dir>${packaging.fess.home.dir}/app</packaging.fess.app.dir>
		<packaging.fess.bin.dir>${packaging.fess.home.dir}/bin</packaging.fess.bin.dir>

index.document.suggest.index=fess
# Index name for crawler documents.
index.document.crawler.index=fess_crawler
# Number of primary shards for crawler queue index.
index.document.crawler.queue.number_of_shards=10
# Number of primary shards for crawler data index.
index.document.crawler.data.number_of_shards=10
# Number of primary shards for crawler filter index.
index.document.crawler.filter.number_of_shards=10
# Number of replicas for crawler queue index.