}
                    }
                } catch (final Exception e) {
                    final String filePath = fileMap.get("path") != null ? fileMap.get("path").toString() : "unknown";
                    final String fileTimestamp = fileMap.get("@timestamp") != null ? fileMap.get("@timestamp").toString() : "unknown";

			<component class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

    /**
     * Adds sitemaps to the thread-local storage.
     * @param sitemaps An array of sitemap URLs.
     */
    public void addSitemaps(final String[] sitemaps) {
        sitemapsLocal.set(sitemaps);
    }

    /**
     * Removes sitemaps from the thread-local storage and returns them.
     * @return An array of sitemap URLs, or null if none were present.
     */
    public String[] removeSitemaps() {

                    facetField = facetResponse.getFieldList().stream().map(field -> {
                        final Map<String, Object> fieldMap = new HashMap<>(2, 1f);
                        fieldMap.put("name", field.getName());
                        fieldMap.put("result", field.getValueCountMap().entrySet().stream().map(e -> {
                            final Map<String, Object> valueCount = new HashMap<>(2, 1f);

### Helpers

- **RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
- **SitemapsHelper**: Sitemap XML parsing, index handling
- **MimeTypeHelper**: MIME detection via Tika
- **EncodingHelper**: Charset detection with BOM
- **UrlConvertHelper**: URL normalization

            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:").append(port).append("/index.html</loc>").append('\n');

      matrix:
        dbversion: [ 'v6.5.0' ]
        go: ['stable', 'oldstable']
        platform: [ ubuntu-latest ]
    runs-on: ${{ matrix.platform }}

    steps:
      - name: Setup TiDB
        uses: Icemap/tidb-action@main
        with:
          port: 9940
          version: ${{matrix.dbversion}}

      - name: Set up Go 1.x
        uses: actions/setup-go@v5
        with:

						<a href="${fe:url('/admin/pathmap/')}" class="nav-link <c:if test="${param.menuType=='pathMapping'}">active</c:if>" <c:if test="${param.menuType=='pathMapping'}">aria-current="page"</c:if>>
							<i class='fa fa-sitemap nav-icon' aria-hidden="true"></i>
							<p><la:message key="labels.menu_path_mapping" /></p>
						</a></li></c:if>
						
					<c:if test="${fe:permission('admin-webauth-view')}">
					<li class="nav-item">

        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {