Crawler - Code Search

src/test/java/org/codelibs/fess/app/web/admin/log/AdminLogActionTest.java

        assertTrue(AdminLogAction.isLogFilename("crawler.log"));
        assertTrue(AdminLogAction.isLogFilename("audit.log"));
        assertTrue(AdminLogAction.isLogFilename("application-2024-01-01.log"));
    }

    @Test
    public void test_isLogFilename_withLogGzExtension() {
        assertTrue(AdminLogAction.isLogFilename("fess.log.gz"));
        assertTrue(AdminLogAction.isLogFilename("crawler.log.gz"));

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Sun Jan 11 08:43:05 GMT 2026

- 10.5K bytes

- Click Count (0)

github.com/codelibs/fess-crawler

fess-crawler-lasta/src/main/resources/crawler/interval.xml

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="intervalController"
		class="org.codelibs.fess.crawler.interval.impl.DefaultIntervalController"
		instance="prototype">
	</component>

Created: Sun Apr 12 03:50:13 GMT 2026

- Last Modified: Sun Oct 11 02:16:55 GMT 2015

- 389 bytes

- Click Count (0)

github.com/codelibs/fess

src/test/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceActionTest.java

        assertTrue(action.isLogFilename("crawler.log"));
        assertTrue(action.isLogFilename("audit.log"));
    }

    @Test
    public void test_isLogFilename_logGzExtension() {
        final AdminMaintenanceAction action = new AdminMaintenanceAction();
        assertTrue(action.isLogFilename("fess.log.gz"));
        assertTrue(action.isLogFilename("crawler.log.gz"));
    }

    @Test

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Sat Mar 28 06:59:19 GMT 2026

- 5.3K bytes

- Click Count (0)

github.com/codelibs/fess

docs/es/README.md

* [LastaFlute](https://github.com/lastaflute/lastaflute "LastaFlute"): Framework Web
* [Lasta Job](https://github.com/lastaflute/lasta-job "Lasta Job"): Programador de Tareas
* [Fess Crawler](https://github.com/codelibs/fess-crawler "Fess Crawler"): Rastreador Web

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Tue Nov 11 22:42:32 GMT 2025

- 7.8K bytes

- Click Count (0)

github.com/codelibs/fess-crawler

fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/entity/OpenSearchAccessResultData.java

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.entity;

import java.io.IOException;
import java.util.Map;

import org.codelibs.core.misc.Base64Util;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

/**

Created: Sun Apr 12 03:50:13 GMT 2026

- Last Modified: Sun Jul 06 02:13:03 GMT 2025

- 3.2K bytes

- Click Count (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/ingest/Ingester.java

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.ingest;

import java.util.Map;

import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.entity.DataStoreParams;
import org.codelibs.fess.util.ComponentUtil;

/**

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Thu Jul 17 08:28:31 GMT 2025

- 4.1K bytes

- Click Count (0)

github.com/codelibs/fess

src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.exception.ChildUrlsException;

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Thu Mar 12 01:46:45 GMT 2026

- 55.3K bytes

- Click Count (0)

github.com/codelibs/fess

src/main/resources/crawler_opensearch+urlQueueService.xml

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<component name="urlQueueService"
		class="org.codelibs.fess.crawler.service.FessUrlQueueService">
		<arg>crawlerConfig</arg>
	</component>

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Thu Oct 30 05:29:49 GMT 2025

- 336 bytes

- Click Count (0)

github.com/codelibs/fess

src/test/resources/run.sh

#!/bin/bash

touch $(ls -d ./fess-*/logs)/fess-crawler.log
tail -f ./fess-*/logs/*.log &

mvn test -P integrationTests -Dtest.fess.url="http://localhost:8080" -Dtest.search_engine.url="http://localhost:9201"
ret=$?

if [ $ret != 0 ] ; then
  for f in `find ./target -type f | grep surefire-reports | grep -v /TEST-` ; do
    cat $f
  done
fi

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Sat Feb 10 03:25:34 GMT 2024

- 353 bytes

- Click Count (0)

github.com/codelibs/fess

src/main/resources/crawler_opensearch+crawlerConfig.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<component name="crawlerConfig" class="org.codelibs.fess.crawler.util.FessCrawlerConfig">
		<property name="queueIndex">"fess_crawler.queue"</property>
		<property name="queueShards">5</property>
		<property name="queueReplicas">1</property>

Created: Tue Mar 31 13:07:34 GMT 2026

- Last Modified: Thu Oct 30 05:29:49 GMT 2025

- 756 bytes

- Click Count (0)

Search Options