Search Options

Results per page
Sort
Preferred Languages
Advance

Results 71 - 80 of 445 for CRAWLER (0.31 sec)

  2. github.com/codelibs/fess-crawler

    fess-crawler/src/test/java/org/codelibs/fess/crawler/rule/impl/SitemapsRuleTest.java 

     */
package org.codelibs.fess.crawler.rule.impl;

import java.io.File;
import java.io.InputStream;
import java.util.regex.Pattern;

import org.codelibs.core.io.CloseableUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.helper.SitemapsHelper;
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Sat Mar 15 06:52:00 UTC 2025
    - 4.7K bytes
    - Viewed (0)
  3. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/client/AbstractCrawlerClient.java 

     */
package org.codelibs.fess.crawler.client;

import java.io.File;
import java.io.IOException;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Sat Sep 06 04:15:37 UTC 2025
    - 9.7K bytes
    - Viewed (10)
  4. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/rule/impl/AbstractRule.java 

     * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.rule.impl;

import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.processor.ResponseProcessor;
import org.codelibs.fess.crawler.rule.Rule;
import org.codelibs.fess.crawler.rule.RuleManager;

import jakarta.annotation.Resource;

/**
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Wed Sep 03 14:42:53 UTC 2025
    - 2.8K bytes
    - Viewed (0)
  5. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/SitemapsResponseProcessor.java 

    import org.codelibs.fess.crawler.entity.Sitemap;
import org.codelibs.fess.crawler.entity.SitemapSet;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.helper.SitemapsHelper;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

import jakarta.annotation.Resource;

/**
 * A response processor implementation that handles sitemaps.
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 3.4K bytes
    - Viewed (0)
  6. github.com/codelibs/fess

    MIGRATION.md 

      -F "file=@documents.bulk"
```

### 4. Direct Crawler Configuration

Instead of importing documents, configure Fess to crawl your data sources:

**Web Crawling**:
- **Admin Path**: Crawler > Web
- **Supports**: HTTP/HTTPS websites
- **Features**: JavaScript rendering, authentication, custom headers

**File Crawling**:
- **Admin Path**: Crawler > File
- **Supports**: SMB, FTP, local file systems
    Registered: Sat Dec 20 09:19:18 UTC 2025
    - Last Modified: Thu Nov 06 12:40:11 UTC 2025
    - 23.2K bytes
    - Viewed (0)
  7. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/extractor/impl/ApiExtractor.java 

    import org.codelibs.core.timer.TimeoutTask;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.client.AccessTimeoutTarget;
import org.codelibs.fess.crawler.client.http.Authentication;
import org.codelibs.fess.crawler.client.http.RequestHeader;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.ExtractException;

import com.google.common.base.Charsets;
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Mon Nov 24 03:59:47 UTC 2025
    - 12.2K bytes
    - Viewed (0)
  8. github.com/codelibs/fess-crawler

    fess-crawler/src/test/java/org/codelibs/fess/crawler/transformer/impl/FileTransformerTest.java 

     */
package org.codelibs.fess.crawler.transformer.impl;

import java.io.File;
import java.io.IOException;
import java.util.Map;

import org.codelibs.core.io.FileUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.AccessResultDataImpl;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Sat Mar 15 06:52:00 UTC 2025
    - 7.6K bytes
    - Viewed (0)
  9. github.com/codelibs/fess-crawler

    fess-crawler/src/test/java/org/codelibs/fess/crawler/extractor/impl/EmlExtractorTest.java 

     */
package org.codelibs.fess.crawler.extractor.impl;

import java.io.IOException;
import java.io.InputStream;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ExtractData;
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Thu Aug 07 02:55:08 UTC 2025
    - 4.6K bytes
    - Viewed (1)
  10. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/util/CrawlingParameterUtil.java 

     * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.util;

import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;

/**
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Sat Nov 22 13:28:22 UTC 2025
    - 6.4K bytes
    - Viewed (0)
  11. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/util/XmlUtil.java 

     *   <li>org.codelibs.fess.crawler.Constants</li>
 *   <li>org.codelibs.fess.crawler.exception.CrawlerSystemException</li>
 *   <li>org.codelibs.fess.crawler.entity.AccessResultData</li>
 *   <li>org.codelibs.core.lang.StringUtil</li>
 * </ul>
 *
 * @see org.codelibs.fess.crawler.Constants
 * @see org.codelibs.fess.crawler.exception.CrawlerSystemException
 * @see org.codelibs.fess.crawler.entity.AccessResultData
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Sat Nov 22 13:28:22 UTC 2025
    - 9.5K bytes
    - Viewed (0)
Back to top