client =
      OkHttpClient
        .Builder()
        .sslSocketFactory(certificates.sslSocketFactory(), certificates.trustManager)
        .build()
  }

  fun run() {
    showUrl("https://squareup.com/robots.txt")
    showUrl("https://publicobject.com/helloworld.txt")
  }

  private fun showUrl(url: String) {
    val request = Builder().url(url).build()
    client
      .newCall(request)
      .execute()

    <magic priority="50">
      <match value="MPX,Microsoft Project for Windows," type="string" offset="0"/>
    </magic>
    <sub-class-of type="text/plain"/>
  </mime-type>
  <mime-type type="text/x-robots">
    <!-- robots.txt file -->
    <!-- draft: https://datatracker.ietf.org/doc/html/draft-koster-rep -->
    <!-- should have a higher priority than rfc822 - TIKA-3489 -->
    <!-- and a lower priority than pcap - TIKA-4174 -->

    peer.acceptFrame() // ACK
    peer.acceptFrame() // SYN_STREAM
    peer.acceptFrame() // DATA
    peer.sendFrame().headers(false, 3, headerEntries("a", "android"))
    peer.sendFrame().data(true, 3, Buffer().writeUtf8("robot"), 5)
    peer.acceptFrame() // PING
    peer.sendFrame().ping(true, Http2Connection.AWAIT_PING, 0) // PING
    peer.play()

    // Play it back.
    val connection = connect(peer)

import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.jspecify.annotations.NullUnmarked;
import org.jspecify.annotations.Nullable;

/**
 * Tests for {@code ForwardingSet}.
 *
 * @author Robert Konigsberg
 * @author Louis Wasserman
 */
@NullUnmarked
public class ForwardingSetTest extends TestCase {
  static class StandardImplForwardingSet<T> extends ForwardingSet<T> {
    private final Set<T> backingSet;

        final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
        if (clientConfigMap != null) {
            paramMap.putAll(clientConfigMap);
        }

        // robots txt enabled
        if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
            paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
        }

        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.jspecify.annotations.NullUnmarked;
import org.jspecify.annotations.Nullable;

/**
 * Tests for {@code ForwardingNavigableMap}.
 *
 * @author Robert Konigsberg
 * @author Louis Wasserman
 */
@NullUnmarked
public class ForwardingNavigableMapTest extends TestCase {
  static class StandardImplForwardingNavigableMap<K, V> extends ForwardingNavigableMap<K, V> {

        assertTrue(directive.allows("/admin/public/page.html"));
        assertFalse(directive.allows("/admin/secret.html"));
    }

    public void test_complexScenario() {
        // Test complex robots.txt scenario
        RobotsTxt robotsTxt = new RobotsTxt();

        // Add directive for all bots
        Directive allBotsDirective = new Directive("*");
        allBotsDirective.addDisallow("/admin/");

    /**
     * Constructs a new EmlExtractor.
     */
    public EmlExtractor() {
        // Default constructor
    }

    /* (non-Javadoc)
     * @see org.codelibs.robot.extractor.Extractor#getText(java.io.InputStream, java.util.Map)
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {

            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());
            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));
            assertFalse(urlFilter.match("http://localhost:7070/websvn/"));
        } finally {
            server.stop();
        }
    }