# Migration Guide

This guide provides comprehensive instructions for migrating from other enterprise search systems to Fess. Fess offers robust compatibility layers, import tools, and APIs to facilitate smooth transitions from various search platforms.

## Table of Contents

- [Supported Migration Sources](#supported-migration-sources)
- [Pre-Migration Planning](#pre-migration-planning)
- [Migration Methods](#migration-methods)

    boolean longWait = NANOSECONDS.toSeconds(thread.timeSpentBlocked) >= 5;
    // Count how long it actually took to return; we'll accept any number between the expected delay
    // and the approximate actual delay, to be robust to variance in thread scheduling.
    char overWaitNanosFirstDigit =
        Long.toString(thread.timeSpentBlocked - MILLISECONDS.toNanos(longWait ? 5000 : 3000))
            .charAt(0);

    boolean longWait = NANOSECONDS.toSeconds(thread.timeSpentBlocked) >= 5;
    // Count how long it actually took to return; we'll accept any number between the expected delay
    // and the approximate actual delay, to be robust to variance in thread scheduling.
    char overWaitNanosFirstDigit =
        Long.toString(thread.timeSpentBlocked - MILLISECONDS.toNanos(longWait ? 5000 : 3000))
            .charAt(0);

				"audio/l20",
				"audio/l24",
				"audio/l8",
				"audio/lpc",
				"audio/midi",
				"audio/mobile-xmf",
				"audio/mp4",
				"audio/mp4a-latm",
				"audio/mpa",
				"audio/mpa-robust",
				"audio/mpeg",
				"audio/mpeg4-generic",
				"audio/ogg",
				"audio/vorbis",
				"audio/x-oggflac",
				"audio/x-oggpcm",
				"audio/opus",
				"audio/speex",
				"audio/parityfec",

labels.crawling_info_thread_dump=Vidage de thread
labels.crawling_info_CrawlerStartTime=Heure de début du robot d'exploration
labels.crawling_info_CrawlerEndTime=Heure de fin du robot d'exploration
labels.crawling_info_CrawlerExecTime=Temps d'exécution du robot d'exploration
labels.crawling_info_CrawlerStatus=Statut du robot d'exploration
labels.crawling_info_WebFsCrawlExecTime=Temps d'exécution de l'exploration (Web/Fichier)

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

            httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

      client
        .newBuilder()
        .eventListener(EventListener.NONE)
        .dns(dohDns)
        .build()

    dohEnabledClient.get("https://www.twitter.com/robots.txt")
    dohEnabledClient.get("https://www.facebook.com/robots.txt")
  }

  @Test
  fun testCustomTrustManager() {
    assumeNetwork()

    val trustManager =
      object : X509TrustManager {

    /** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */