USER - Code Search

fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/RobotsTxt.java

 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Jul 06 02:13:03 UTC 2025

- 10K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/test/resources/org/codelibs/fess/crawler/helper/robots.txt

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Oct 11 02:16:55 UTC 2015

- 566 bytes

- Viewed (0)

github.com/codelibs/fess-suggest

src/main/java/org/codelibs/fess/suggest/constants/SuggestConstants.java

    public static final String EMPTY_STRING = "";

    /** The system property name for user dictionary encoding. */
    public static final String USER_DICT_ENCODING = "fess.user.dict.encoding";

    /** The system property name for user dictionary path. */
    public static final String USER_DICT_PATH = "fess.user.dict.path";

    /** The text separator. */
    public static final String TEXT_SEPARATOR = " ";

Registered: Fri Sep 19 09:08:11 UTC 2025

- Last Modified: Fri Jul 04 14:00:23 UTC 2025

- 1.5K bytes

- Viewed (0)

github.com/codelibs/fess-suggest

README.md

- **Asynchronous Operations**: Non-blocking suggestion requests with callback support
- **Index Management**: Automatic index creation, switching, and maintenance
- **Customizable Scoring**: User boost, document frequency, and query frequency weighting

## Technology Stack

- **Java**: 21+ (configured via parent POM)
- **OpenSearch**: Latest (provided scope)
- **Apache Lucene**: Query parsing and text analysis

Registered: Fri Sep 19 09:08:11 UTC 2025

- Last Modified: Sun Aug 31 03:31:14 UTC 2025

- 12.1K bytes

- Viewed (1)

github.com/codelibs/fess-suggest

src/test/java/org/codelibs/fess/suggest/entity/SuggestItemTest.java

        // Test Kind enum
        assertEquals("query", SuggestItem.Kind.QUERY.toString());
        assertEquals("document", SuggestItem.Kind.DOCUMENT.toString());
        assertEquals("user", SuggestItem.Kind.USER.toString());
    }

    @Test
    public void testGetUpdatedSource() {
        // Test getUpdatedSource method
        String[] text = { "test" };
        String[][] readings = { { "test" } };

Registered: Fri Sep 19 09:08:11 UTC 2025

- Last Modified: Mon Sep 01 13:33:03 UTC 2025

- 16.7K bytes

- Viewed (0)

github.com/codelibs/fess-suggest

src/main/java/org/codelibs/fess/suggest/entity/SuggestItem.java

    /**
     * The kind of suggest item.
     */
    public enum Kind {
        /** Document kind. */
        DOCUMENT("document"),
        /** Query kind. */
        QUERY("query"),
        /** User kind. */
        USER("user");

        private final String kind;

        /**
         * Constructor for Kind.
         * @param kind The kind string.
         */
        Kind(final String kind) {

Registered: Fri Sep 19 09:08:11 UTC 2025

- Last Modified: Thu Aug 07 02:41:28 UTC 2025

- 25.1K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/test/resources/extractor/eml/sample2.eml

<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, minimum-scale=1.0, maximum-scale=1.0, user-scalable=0" />
<meta name="apple-mobile-web-app-capable" content="yes" />
<style type="text/css">

@media only screen and (max-width: 420px) {
a[class="article-headline"] {
font-size:22px !important;

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sat Jan 16 07:50:35 UTC 2016

- 91.6K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/main/java/org/codelibs/fess/crawler/extractor/impl/ApiExtractor.java

    public void setAuthSchemeProviderMap(final Map<String, AuthSchemeProvider> authSchemeProviderMap) {
        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Thu Aug 07 02:55:08 UTC 2025

- 12.2K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/RobotsTxtHelper.java

 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Jul 06 02:13:03 UTC 2025

- 7.7K bytes

- Viewed (0)

github.com/codelibs/fess-suggest

src/test/java/org/codelibs/opensearch/extension/kuromoji/index/analysis/KuromojiTokenizerFactory.java

        String[] values = CSVUtil.parse(rule);
        if (dup.add(values[0]) == false) {
            throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary.");
        }
        return rule;
    }

    public static UserDictionary getUserDictionary(Environment env, Settings settings) {

Registered: Fri Sep 19 09:08:11 UTC 2025

- Last Modified: Sat Mar 15 06:51:20 UTC 2025

- 4.7K bytes

- Viewed (0)

Search Options