// Configure crawling parameters
        crawler.addUrl("https://example.com");
        crawler.crawlerContext.setMaxAccessCount(100);
        crawler.crawlerContext.setNumOfThread(5);
        crawler.urlFilter.addInclude("https://example.com/.*");
        
        // Execute crawling
        String sessionId = crawler.execute();
        System.out.println("Crawling completed. Session ID: " + sessionId);
    }
}

    public void setSessionIdList(final List<String> sessionIdList) {
        this.sessionIdList = sessionIdList;
    }

    /**
     * Sets the flag indicating whether crawling should be finished.
     *
     * @param finishCrawling true if crawling should be finished, false otherwise
     */
    public void setFinishCrawling(final boolean finishCrawling) {
        this.finishCrawling = finishCrawling;
    }

    /**

        ComponentUtil.register(systemHelper, "systemHelper");
        ComponentUtil.setFessConfig(fessConfig);

        // Execute
        String result = purgeLogJob.execute();

        // Assert crawling info and status update were called
        assertTrue(deleteCrawlingInfoCalled[0]);
        assertTrue(updateJobLogStatusCalled[0]);

        // Assert other services were not called

        crawlerContext.setIntervalController(new TestIntervalController());

        // Start crawling
        crawlerContext.setStatus(CrawlerStatus.RUNNING);
        crawlerContext.setActiveThreadCount(5);

        // Simulate crawling progress
        for (int i = 0; i < 100; i++) {
            crawlerContext.incrementAndGetAccessCount();

		// hence not included in the collector path.
		pathComponents = pathComponents[:bucketIdx]
	}

	innerHandler := h.handle(pathComponents, isListingRequest, buckets)

	// Add tracing to the prom. handler
	tracedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
		if ok {

    public void test_init_withIngestFactory() {
        ComponentUtil.register(ingestFactory, "ingestFactory");
        indexUpdater.init();
        assertNotNull(indexUpdater);
    }

    // Test destroy when crawling is not finished
    public void test_destroy_notFinished() {
        final List<Crawler> crawlerList = new ArrayList<>();
        final TestCrawler crawler = new TestCrawler();
        crawlerList.add(crawler);

https://hackernoon.com/explore-how-to-effectively-use-jwt-with-fastapi title: Explore How to Effectively Use JWT With FastAPI - author: Nicoló Lino author_link: https://www.nlino.com link: https://github.com/softwarebloat/python-tracing-demo title: Instrument FastAPI with OpenTelemetry tracing and visualize traces in Grafana Tempo. - author: Mikhail Rozhkov, Elena Samuylova author_link: https://www.linkedin.com/in/mnrozhkov/ link: https://www.evidentlyai.com/blog/fastapi-tutorial title: ML serving and monitoring...

        // We just verify that the crawl completes with EXIT_OK
        assertTrue("Crawler should complete", result == Constants.EXIT_OK);
    }

    // Test concurrent crawling
    public void test_doCrawl_concurrentWebAndData() {
        final List<String> executionOrder = Collections.synchronizedList(new ArrayList<>());

        WebFsIndexHelper mockWebHelper = new WebFsIndexHelper() {

        transformer.addTransformationRule("<[^>]+>", ""); // Remove HTML tags
        transformer.addTransformationRule("\\s+", " "); // Normalize whitespace

        // Simulate crawling response
        ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/page.html");
        responseData.setParentUrl("http://example.com/");

        regarding when retracing needs to occur by implementing the Tracing
        Protocol available through
        `tf.types.experimental.SupportsTracingProtocol`.
    *   `TypeSpec` classes (as associated with `ExtensionTypes`) also implement
        the Tracing Protocol which can be overridden if necessary.
    *   The newly introduced `reduce_retracing` option also uses the Tracing