content2 - Code Search

fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java

        return clientFactory.getClient(url);
    }

    /**
     * Checks if the content has been updated since the last crawl.
     * @param client The crawler client.
     * @param urlQueue The URL queue entry.
     * @return true if content is updated, false otherwise.
     */
    protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) {

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Thu Aug 07 02:55:08 UTC 2025

- 20.4K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/test/java/org/codelibs/fess/crawler/transformer/impl/HtmlTransformerTest.java

        String content;

        content = "...;charset=UTF-8\"...";
        assertEquals("UTF-8", htmlTransformer.parseCharset(content));

        content = "...; charset=UTF-8\"...";
        assertEquals("UTF-8", htmlTransformer.parseCharset(content));

        content = "...;charset = UTF-8\"...";
        assertEquals("UTF-8", htmlTransformer.parseCharset(content));

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sat Mar 15 06:52:00 UTC 2025

- 13.8K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/test/java/org/codelibs/fess/crawler/transformer/TransformerTest.java

                byte[] bytes = is.readAllBytes();
                String content = new String(bytes);
                for (Map.Entry<String, String> rule : transformationRules.entrySet()) {
                    content = content.replaceAll(rule.getKey(), rule.getValue());
                }
                resultData.setData(content.getBytes());
            } catch (Exception e) {
                throw new RuntimeException(e);

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sat Sep 06 04:15:37 UTC 2025

- 28K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/main/java/org/codelibs/fess/crawler/extractor/ExtractorBuilder.java

 * and cache file size to optimize the extraction process.
 *
 * <p>
 * The main purpose of this class is to simplify the extraction process by providing a fluent interface
 * for configuring the extraction parameters and handling the underlying complexities of content processing,
 * such as MIME type detection, extractor selection, and content length validation.
 * </p>
 *
 * <p>

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Jul 06 02:13:03 UTC 2025

- 10.1K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/AccessResult.java

    /**
     * Returns the content length of the accessed resource.
     *
     * @return the content length
     */
    Long getContentLength();

    /**
     * Sets the content length of the accessed resource.
     *
     * @param contentLength the content length
     */
    void setContentLength(Long contentLength);

    /**

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Jul 06 02:13:03 UTC 2025

- 5.2K bytes

- Viewed (10)

github.com/codelibs/fess-crawler

fess-crawler/src/test/resources/extractor/test_sjis.html

<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS"/>
<title>�^�C�g��</title>
</head>
<body>
�e�X�g
</body>

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Oct 11 02:16:55 UTC 2015

- 149 bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/main/java/org/codelibs/fess/crawler/client/http/form/FormScheme.java

    /**
     * Extracts the token value from the content using the given pattern.
     * @param tokenPattern The regex pattern.
     * @param content The content to search.
     * @return The extracted token value.
     */
    protected String getTokenValue(final String tokenPattern, final String content) {
        final Matcher matcher = Pattern.compile(tokenPattern).matcher(content);
        if (matcher.find()) {

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Jul 06 02:13:03 UTC 2025

- 14.3K bytes

- Viewed (1)

github.com/codelibs/fess-crawler

fess-crawler/src/test/resources/extractor/test_attr.html

<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>タイトル</title>
</head>
<body>
本文1
<img src="hoge.gif" alt="画像1"/>
<img src="hoge.gif" alt="画像2" title="タイトル1"/>
<a href="hoge.html" title="タイトル2">リンク1</a>
本文2
</body>

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sun Oct 11 02:16:55 UTC 2015

- 308 bytes

- Viewed (0)

github.com/codelibs/fess-suggest

src/main/java/org/codelibs/fess/suggest/index/contents/document/DocumentReader.java

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.suggest.index.contents.document;

import java.io.Closeable;
import java.util.Map;

/**
 * Interface for reading documents and extracting their contents into a map.
 * Implementations of this interface should provide the logic for reading
 * documents and converting them into a key-value structure.
 *

Registered: Fri Sep 19 09:08:11 UTC 2025

- Last Modified: Fri Jul 04 14:00:23 UTC 2025

- 1.4K bytes

- Viewed (0)

github.com/codelibs/fess-crawler

fess-crawler/src/main/java/org/codelibs/fess/crawler/client/AbstractCrawlerClient.java

    public static final String ACCESS_TIMEOUT_PROPERTY = "accessTimeout";

    /** The property name for maximum content length. */
    public static final String MAX_CONTENT_LENGTH = "maxContentLength";

    /** The property name for maximum cached content size. */
    public static final String MAX_CACHED_CONTENT_SIZE = "maxCachedContentSize";

    /** The crawler container. */
    @Resource

Registered: Sun Sep 21 03:50:09 UTC 2025

- Last Modified: Sat Sep 06 04:15:37 UTC 2025

- 9.7K bytes

- Viewed (10)

Search Options