# Test file for malformed robots.txt parsing
# This file contains various malformed directives that should be handled gracefully

# Case 1: Directives before any User-agent (should be ignored)
Disallow: /orphaned1/
Allow: /orphaned2/

# Case 2: Valid user-agent with various malformed directives
User-agent: GoodBot
Disallow: /admin/
InvalidDirective: some-value
unknown-field: test
Disallow /missing-colon

              .build())
      .build();

  public void run() throws Exception {
    Request request = new Request.Builder()
        .url("https://publicobject.com/robots.txt")
        .build();

    try (Response response = client.newCall(request).execute()) {
      if (!response.isSuccessful()) throw new IOException("Unexpected code " + response);

            tempDir.delete();
            tempDir.mkdirs();

            // robots.txt
            StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.jspecify.annotations.NullUnmarked;

/**
 * Tests for {@link ForwardingCollection}.
 *
 * @author Robert Konigsberg
 * @author Hayward Chan
 * @author Louis Wasserman
 */
@NullUnmarked
public class ForwardingCollectionTest extends TestCase {

import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.jspecify.annotations.NullUnmarked;
import org.jspecify.annotations.Nullable;

/**
 * Tests for {@code ForwardingQueue}.
 *
 * @author Robert Konigsberg
 * @author Louis Wasserman
 */
@NullUnmarked
public class ForwardingQueueTest extends TestCase {

  static final class StandardImplForwardingQueue<T> extends ForwardingQueue<T> {

          handshakeCertificates.trustManager,
        )
    }

    val client = clientBuilder.build()

    val request =
      Request
        .Builder()
        .url("https://valid-isrgrootx1.letsencrypt.org/robots.txt")
        .build()
    client.newCall(request).execute().use { response ->
      assertThat(response.code).isEqualTo(404)
      assertThat(response.protocol).isEqualTo(Protocol.HTTP_2)
    }
  }

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * RobotsTxtException is an exception class that represents an error related to robots.txt processing during web crawling.
 * It extends CrawlerSystemException and provides constructors to create instances with a message and/or a cause.
 *
 */
public class RobotsTxtException extends CrawlerSystemException {

    OkHttpDebugLogging.enable("org.bouncycastle.jsse")
    platform.assumeBouncyCastle()
  }

  @Test
  fun testMozilla() {
    assumeNetwork()

    val request = Request.Builder().url("https://mozilla.org/robots.txt").build()

    client.newCall(request).execute().use {
      assertThat(it.protocol).isEqualTo(Protocol.HTTP_2)
      assertThat(it.handshake!!.tlsVersion).isEqualTo(TlsVersion.TLS_1_3)
    }
  }

        .cache(Cache(FakeFileSystem(), "/cache".toPath(), 10_000_000))
        .build()
  }

  @Test
  fun testRequestExternal() {
    assumeNetwork()

    val request = Request("https://www.google.com/robots.txt".toHttpUrl())

    val networkRequest =
      request
        .newBuilder()
        .build()

    val call = client.newCall(networkRequest)

    call.execute().use { response ->

import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.jspecify.annotations.NullUnmarked;
import org.jspecify.annotations.Nullable;

/**
 * Tests for {@code ForwardingList}.
 *
 * @author Robert Konigsberg
 * @author Louis Wasserman
 */
@NullUnmarked
public class ForwardingListTest extends TestCase {
  static final class StandardImplForwardingList<T> extends ForwardingList<T> {