<image file="taro.png" />
        </item>
        <item sex="male" custid="B23456">
                <name>佐藤二朗</name>
                <access kind="email">******@****.***.bar</access>
                <image file="jiro.png" />
        </item>
        <item sex="female" custid="C34567">
                <name>田中花子</name>
                <access kind="tel">090-xxxx-xxxx</access>

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package com.foo.bar;

public class Test {
    public String test="テスト";

            assertTrue(robotsTxt.allows("/help.html", userAgent));
            assertTrue(robotsTxt.allows("/help/faq.html", userAgent));
            assertTrue(robotsTxt.allows("/foo/bar/", userAgent));
            assertTrue(robotsTxt.allows("/foo/bar/index.html", userAgent));
            assertEquals(0, robotsTxt.getCrawlDelay(userAgent));
        }

        for (String userAgent : new String[] { "BruteBot", "FOO BruteBot/1.0" }) {

// Test Java Script

function foo() {
	var bar = "テスト";

                + "<field name=\"image\"><list><item>taro.png</item><item>jiro.png</item><item>hanako.png</item></list></field>\n"//
                + "<field name=\"email\"><list><item></item><item>******@****.***.bar</item></list></field>\n"//
                + "<field name=\"url\">http://www.taro.com/</field>\n"//

     *
     * @param c the character to check
     * @return {@code true} if the character is a valid URL character, {@code false} otherwise
     */
    public static boolean isUrlChar(final char c) {
        if (c >= 'a' && c <= 'z' //
                || c >= 'A' && c <= 'Z' //
                || c >= '0' && c <= '9' //
                || c == '.' //
                || c == '-' //
                || c == '*' //

    public static String stripInvalidXMLCharacters(final String in) {
        if (StringUtil.isEmpty(in)) {
            return in;
        }

        final StringBuilder buf = new StringBuilder(in.length());
        char c;
        for (int i = 0; i < in.length(); i++) {
            c = in.charAt(i);
            if (c == 0x9 || c == 0xA || c == 0xD || c >= 0x20 && c <= 0xD7FF || c >= 0xE000 && c <= 0xFFFD

                    || PRIORITY_ELEMENT.equals(qName)) {
                buf = new StringBuilder();
            }
        }

        @Override
        public void characters(final char[] ch, final int offset, final int length) {
            if (buf != null) {
                buf.append(new String(ch, offset, length));
            }
        }

        @Override

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

		<hoge:access kind="url">http://www.taro.com/</hoge:access>
		<hoge:image file="taro.png" />
	</hoge:item>
	<hoge:item sex="male" custid="B23456">
		<hoge:name>佐藤二朗</hoge:name>
		<hoge:access kind="email">******@****.***.bar</hoge:access>
		<hoge:image file="jiro.png" />
	</hoge:item>
	<hoge:item sex="female" custid="C34567">
		<hoge:name>田中花子</hoge:name>
		<hoge:access kind="tel">090-xxxx-xxxx</hoge:access>
		<hoge:image file="hanako.png" />