¿Quieres solo tener un `str`? ¿O solo un `dict`? ¿O un instance de clase modelo de base de datos directamente? Todo funciona de la misma manera.

¿En realidad no tienes usuarios que inicien sesión en tu aplicación sino robots, bots u otros sistemas, que solo tienen un token de acceso? Una vez más, todo funciona igual.

        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        assertTrue(directive.allows("/admin/public/page.html"));
        assertFalse(directive.allows("/admin/secret.html"));
    }

    public void test_complexScenario() {
        // Test complex robots.txt scenario
        RobotsTxt robotsTxt = new RobotsTxt();

        // Add directive for all bots
        Directive allBotsDirective = new Directive("*");
        allBotsDirective.addDisallow("/admin/");

    /**
     * Constructs a new EmlExtractor.
     */
    public EmlExtractor() {
        // Default constructor
    }

    /* (non-Javadoc)
     * @see org.codelibs.robot.extractor.Extractor#getText(java.io.InputStream, java.util.Map)
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {

            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());
            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));
            assertFalse(urlFilter.match("http://localhost:7070/websvn/"));
        } finally {
            server.stop();
        }
    }

        .addHeader("Cookie: s=square")
        .addHeader("Cookie: a=android")
        .addHeader("Cookies: delicious")
    response.setHeader("cookie", "r=robot")
    assertThat(headersToList(response))
      .containsExactly("Cookies: delicious", "cookie: r=robot")
  }

  @Test
  fun mockResponseSetHeaders() {
    val response =
      MockResponse()
        .clearHeaders()

      HandshakeCertificates
        .Builder()
        .heldCertificate(certificate)
        .build()
    server.useHttps(serverHandshakeCertificates.sslSocketFactory())
    url = server.url("/robots.txt")
  }

  /**
   * Test connecting to the main host then an alternative, although only subject alternative names
   * are used if present no special consideration of common name.
   */
  @Test

        .addHeader("Cookie: s=square")
        .addHeader("Cookie: a=android")
        .addHeader("Cookies: delicious")
    builder.setHeader("cookie", "r=robot")
    assertThat(headersToList(builder)).containsExactly("Cookies: delicious", "cookie: r=robot")
  }

  @Test
  fun mockResponseSetHeaders() {
    val builder =
      MockResponse
        .Builder()
        .clearHeaders()

# Pattern for environment variable keys in crawling data.
crawler.data.env.param.key.pattern=^FESS_ENV_.*
# Whether to ignore robots.txt during crawling.
crawler.ignore.robots.txt=false
# Whether to ignore robots meta tags during crawling.
crawler.ignore.robots.tags=false
# Whether to ignore content exceptions during crawling.
crawler.ignore.content.exception=true
# HTTP status codes considered as failure URLs.

            CrawlingParameterUtil.setDataService(null);
        }
        log(logHelper, LogType.FINISHED_THREAD, crawlerContext);
    }

    /**
     * Adds sitemaps from robots.txt to the crawling queue.
     * @param urlQueue The URL queue to add sitemaps to.
     */
    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {