"名詞-非自立-助動詞語幹",
          "名詞-非自立-副詞可能",
          "名詞-副詞可能",
          "連体詞"
        ]
      },
      "stopword_en_filter": {
        "type": "stop",
        "stopwords": "_english_"
      },
      "content_length_filter": {
        "type": "length",
        "max": 30
      },
      "limit_token_count_filter": {
        "type": "limit",
        "max_token_count": 10000

     */
    @SuppressWarnings("unchecked")
    protected Map<String, Object>[] getFromArrayIndex(final String index, final String type, final String key) {
        final String actualIndex = index + "." + type.toLowerCase(Locale.ENGLISH);
        try {
            SearchResponse response = client.prepareSearch()
                    .setIndices(actualIndex)
                    .setScroll(settings.getScrollTimeout())

     * @return This builder instance.
     */
    public SuggestSettingsBuilder setSettingsIndexName(final String settingsIndexName) {
        this.settingsIndexName = settingsIndexName.toLowerCase(Locale.ENGLISH);
        return this;
    }

    /**
     * Adds an initial setting.
     * @param key The key of the setting.
     * @param value The value of the setting.
     * @return This builder instance.
     */

     */
    public Crawler() {
        crawlerContext = new CrawlerContext();
        final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssSSS", Locale.ENGLISH);
        crawlerContext.sessionId = sdf.format(new Date());
    }

    /**
     * Adds a URL to the crawling queue.
     * @param url The URL to add.
     */
    public void addUrl(final String url) {

                        currentDirectiveList.clear();
                        isGroupRecordStarted = false;
                    }
                    final String userAgent = value.toLowerCase(Locale.ENGLISH);
                    Directive currentDirective = robotsTxt.getDirective(userAgent);
                    if (currentDirective == null) {
                        currentDirective = new Directive(userAgent);

    @Test
    public void testAnalyzeWithDifferentLanguages() {
        // Test analyze with different language settings
        String text = "hello world";
        String field = "content";

        // Test with English
        List<AnalyzeToken> tokensEn = analyzer.analyze(text, field, "en");
        assertNotNull(tokensEn);
        assertEquals(2, tokensEn.size());

        // Test with Japanese

     * @return The parsed date, or null if parsing fails
     */
    protected Date parseLastModifiedDate(final String value) {
        final SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z", Locale.ENGLISH);
        try {
            return sdf.parse(value);
        } catch (final ParseException e) {
            return null;
        }
    }

    /**