import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.opensearch.config.exbhv.RelatedQueryBhv;
import org.codelibs.fess.opensearch.config.exentity.RelatedQuery;
import org.codelibs.fess.util.ComponentUtil;

            switch (c) {
            case '?':
                buf.append("%3f");
                break;
            case '#':
                buf.append("%23");
                break;
            case '@':
                buf.append("%40");
                break;
            case ':':
                buf.append("%3a");
                break;
            case '/':
                buf.append("%2f");

 */
package org.codelibs.fess.crawler.entity;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.query;

import static org.codelibs.core.stream.StreamUtil.split;
import static org.codelibs.core.stream.StreamUtil.stream;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.text.StringEscapeUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.exception.IORuntimeException;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.curl.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.annotation.Secured;

            switch (method) {
            case GET:
                hasGet = true;
                break;
            case POST:
                hasPost = true;
                break;
            case PUT:
                hasPut = true;
                break;
            case DELETE:
                hasDelete = true;
                break;
            case HEAD:
                hasHead = true;

import java.net.URI;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.codelibs.core.exception.InterruptedRuntimeException;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.lang.SystemUtil;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

import javax.naming.directory.Attribute;
import javax.naming.directory.BasicAttribute;

import org.codelibs.core.exception.ClassNotFoundRuntimeException;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.core.misc.Tuple3;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.fess.Constants;
import org.codelibs.fess.helper.PermissionHelper;

    }

    public void test_caseInsensitiveUserAgent() {
        // Test case-insensitive user agent matching
        RobotsTxt robotsTxt = new RobotsTxt();

        Directive directive = new Directive("googlebot");
        directive.addDisallow("/private/");
        robotsTxt.addDirective(directive);

        // Should match case-insensitively
        assertFalse(robotsTxt.allows("/private/", "GoogleBot"));

        try {
            switch (type) {
            case PRE_PROCESSING:
                delayBeforeProcessing();
                break;
            case POST_PROCESSING:
                delayAfterProcessing();
                break;
            case NO_URL_IN_QUEUE:
                delayAtNoUrlInQueue();
                break;
            case WAIT_NEW_URL:
                delayForWaitingNewUrl();