urlQueue2.setMethod("GET");
        urlQueue2.setSessionId(sessionId);
        urlQueue2.setUrl("http://www.example.com/page1"); // Duplicate
        urlQueueList.add(urlQueue2);

        urlQueueService.offerAll(sessionId, urlQueueList);

        // Should only store one item (duplicates are filtered)
        int count = 0;
        while (true) {
            final OpenSearchUrlQueue polled = urlQueueService.poll(sessionId);

        assertNotNull(invalidKeywords);
        assertEquals(0, invalidKeywords.size());
    }

    @Test
    public void testGetKeywordsWithDuplicates() {
        // Test that duplicates are removed
        String query = "test test query";
        String[] fields = { "content" };
        List<String> keywords = SuggestUtil.getKeywords(query, fields);
        assertNotNull(keywords);

    /**
     * Displays the initial duplicate host management page.
     *
     * @param form the search form
     * @return HTML response for the duplicate host list page
     */
    @Execute
    @Secured({ ROLE, ROLE + VIEW })
    public HtmlResponse index(final SearchForm form) {
        return asListHtml();
    }

    /**
     * Displays the duplicate host list with pagination.
     *

    /** List of duplicate host rules for URL conversion */
    protected List<DuplicateHost> duplicateHostList;

    /**
     * Default constructor for DuplicateHostHelper.
     * Creates a new duplicate host helper instance.
     */
    public DuplicateHostHelper() {
        // Default constructor
    }

    /**
     * Initializes the duplicate host helper after construction.

 *   <li>Optionally removing duplicate terms based on a flag.</li>
 *   <li>Limiting the maximum size of alphanumeric and symbol terms.</li>
 * </ul>
 *
 * <p>The {@link TextNormalizeContext} class provides a fluent API to configure the text
 * normalization process, including setting initial buffer capacity, maximum term sizes,
 * duplicate term removal, and custom space characters.
 *
 * <p>Example usage:

        return fessConfig.getCrawlerDocumentMaxSymbolTermSizeAsInteger();
    }

    /**
     * Checks if duplicate term removal is enabled in configuration.
     *
     * @return true if duplicate terms should be removed, false otherwise
     */
    protected boolean isDuplicateTermRemoved() {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();

     */
    public void setInitialBufferSize(final int initialBufferSize) {
        this.initialBufferSize = initialBufferSize;
    }

    /**
     * Sets whether duplicated terms are replaced.
     * @param replaceDuplication If true, duplicated terms are replaced.
     */
    public void setReplaceDuplication(final boolean replaceDuplication) {
        this.replaceDuplication = replaceDuplication;
    }

    /**

# Maximum size of alphanumeric terms in documents.
crawler.document.max.alphanum.term.size=20
# Maximum size of symbol terms in documents.
crawler.document.max.symbol.term.size=10
# Whether to remove duplicate terms in documents.
crawler.document.duplicate.term.removed=false
# Unicode space characters for document parsing.

            if (arMap.containsKey(accessResult.getUrl())) {
                throw new CrawlerSystemException(
                        "AccessResult for URL '" + accessResult.getUrl() + "' already exists. Duplicate URLs are not allowed.");
            }
            arMap.put(accessResult.getUrl(), accessResult);
        }

    }

    /**
     * Gets the count of access results for the specified session.
     *

                    new ContentNotFoundException(urlQueue.getParentUrl(), url));
        }
    }

    /**
     * Stores a child URL in the crawling queue with duplicate host handling.
     * This method applies duplicate host conversion before storing the URL.
     *
     * @param childUrl the child URL to store
     * @param parentUrl the parent URL that referenced this child URL