import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

     * command. Even though search engine crawlers may consider this information
     * when making decisions, they may crawl pages marked "hourly" less
     * frequently than that, and they may crawl pages marked "yearly" more
     * frequently than that. Crawlers may periodically crawl pages marked
     * "never" so that they can handle unexpected changes to those pages.
     */
    private String changefreq;

    /**

    @Size(max = 200)
    public String name;

    /** The description of the file configuration (maximum 1000 characters). */
    @Size(max = 1000)
    public String description;

    /** The file paths to crawl (required, must be valid file URIs). */
    @Required
    @UriType(protocolType = ProtocolType.FILE)
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String paths;

 * This form extends CreateForm to include fields necessary for updating existing data config entries,
 * including tracking information for optimistic locking and audit trails.
 * Data configs define how to crawl and extract data from databases, CSV files, and other data sources.
 *
 */
public class EditForm extends CreateForm {

    /**
     * Creates a new EditForm instance.
     */
    public EditForm() {

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="relatedQuery"/>
    </jsp:include>
    <div class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

    /**
     * Default constructor.
     */
    public SearchBody() {
        super();
    }

    /** Name of the web crawling configuration */
    public String name;

    /** URLs to crawl */
    public String urls;

    /** Description of the web crawling configuration */
    public String description;

    /**
     * Default constructor.
     */
    public SearchBody() {
        super();
    }

    /** Name of the file crawling configuration */
    public String name;

    /** File paths to crawl */
    public String paths;

    /** Description of the file crawling configuration */
    public String description;

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="pathMapping"/>
    </jsp:include>
    <div class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

import java.util.LinkedHashSet;
import java.util.Set;
import java.util.function.Function;

import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * This class represents the result data of a crawl.
 */
public class ResultData implements Serializable {
    private static final long serialVersionUID = 1L;

    /** The name of the transformer. */
    protected String transformerName;