## -*-Ruby-*- $Id: test.rb,v 1.2 2005/11/12 01:14:52 nabeken Exp $
## this file is written in eucJP

load '../bsfilter/bsfilter'
require 'test/unit'
require 'fileutils'

$default_options = ["--homedir", ".", "-v", "-d", "--show-process"]

class DummyFH
  def initialize
    @buf = Array::new
  end
  attr_accessor :buf
  def sync=(*args)
  end
  def print(*arg)
    @buf.push(*arg.flatten.dup)
  end
  def printf(format, *args)
    @buf.push(sprintf(format, *args))
  end
end

class Bsfilter
  attr_accessor :options
  def use_dummyfh
    options["message-fh"] = DummyFH::new
    options["pipe-fh"] = DummyFH::new
  end

  def grep_message(pattern)
    options["message-fh"].buf.grep(pattern)
  end

  def count_message(pattern)
    grep_message(pattern).length
  end

  def grep_pipe(pattern)
    options["pipe-fh"].buf.grep(pattern)
  end

  def count_pipe(pattern)
    grep_pipe(pattern).length
  end
end


def safe_require(file)
  begin
    require file
    return true
  rescue LoadError
    return false
  end
end

def unlink_all
  unlink_prob_sdbm({:force => true})
  unlink_token_sdbm({:force => true})

  unlink_prob_gdbm({:force => true})
  unlink_token_gdbm({:force => true})

  unlink_prob_bdb1({:force => true})
  unlink_token_bdb1({:force => true})

  unlink_prob_bdb({:force => true})
  unlink_token_bdb({:force => true})

  unlink_prob_qdbm({:force => true})
  unlink_token_qdbm({:force => true})
end

def unlink_prob_sdbm(options = {})
    FileUtils.rm(["C.prob.sdbm.dir",
                  "C.prob.sdbm.pag",
                  "ja.prob.sdbm.dir",
                  "ja.prob.sdbm.pag"], options)
end
def unlink_token_sdbm(options = {})
    FileUtils.rm(["C.clean.sdbm.dir",
                  "C.clean.sdbm.pag",
                  "C.spam.sdbm.dir",
                  "C.spam.sdbm.pag",
                  "ja.clean.sdbm.dir",
                  "ja.clean.sdbm.pag",
                  "ja.spam.sdbm.dir",
                  "ja.spam.sdbm.pag"], options)
end


def unlink_prob_gdbm(options = {})
    FileUtils.rm(["C.prob.gdbm",
                  "ja.prob.gdbm"], options)
end
def unlink_token_gdbm(options = {})
    FileUtils.rm(["C.clean.gdbm",
                  "ja.clean.gdbm",
                  "C.spam.gdbm",
                  "ja.spam.gdbm"], options)
end

def unlink_token_bdb1(options = {})
    FileUtils.rm(["C.clean.bdb1",
                  "ja.clean.bdb1",
                  "C.spam.bdb1",
                  "ja.spam.bdb1"], options)
end
def unlink_prob_bdb1(options = {})
    FileUtils.rm(["C.prob.bdb1",
                  "ja.prob.bdb1"], options)
end

def unlink_token_bdb(options = {})
    FileUtils.rm(["C.clean.bdb",
                  "ja.clean.bdb",
                  "C.spam.bdb",
                  "ja.spam.bdb"], options)
end
def unlink_prob_bdb(options = {})
    FileUtils.rm(["C.prob.bdb",
                  "ja.prob.bdb"], options)
end

def unlink_token_qdbm(options = {})
    FileUtils.rm(["C.clean.qdbm",
                  "ja.clean.qdbm",
                  "C.spam.qdbm",
                  "ja.spam.qdbm"], options)
end
def unlink_prob_qdbm(options = {})
    FileUtils.rm(["C.prob.qdbm",
                  "ja.prob.qdbm"], options)
end


class TestMultipleInstances < Test::Unit::TestCase
  def test_by_mbox
    @files = ["testcases/mbox"]

    @bsfilter0 = Bsfilter::new
    @bsfilter0.setup($default_options + ["--mbox"])
    @bsfilter0.use_dummyfh

    @bsfilter1 = Bsfilter::new
    @bsfilter1.setup($default_options)
    @bsfilter1.use_dummyfh

    @bsfilter2 = Bsfilter::new
    @bsfilter2.setup($default_options + ["--mbox"])
    @bsfilter2.use_dummyfh

    @bsfilter3 = Bsfilter::new
    @bsfilter3.setup($default_options)
    @bsfilter3.use_dummyfh

    @bsfilter0.run(@files)
    @bsfilter1.run(@files)
    @bsfilter2.run(@files)
    @bsfilter3.run(@files)

    assert_equal(3, @bsfilter0.count_message(/^file/), "@bsfilter0")
    assert_equal(1, @bsfilter1.count_message(/^file/), "@bsfilter1")
    assert_equal(3, @bsfilter2.count_message(/^file/), "@bsfilter2")
    assert_equal(1, @bsfilter3.count_message(/^file/), "@bsfilter3")
  end

  def test_by_jtokenizer
    return if (! safe_require('MeCab'))

    @files = ["testcases/iso_2022_jp_plain"]

    @bsfilter0 = Bsfilter::new
    @bsfilter0.setup($default_options + ["--jtokenizer", "bigram"])
    @bsfilter0.use_dummyfh

    @bsfilter1 = Bsfilter::new
    @bsfilter1.setup($default_options + ["--jtokenizer", "mecab"])
    @bsfilter1.use_dummyfh

    @bsfilter2 = Bsfilter::new
    @bsfilter2.setup($default_options + ["--jtokenizer", "bigram"])
    @bsfilter2.use_dummyfh

    @bsfilter3 = Bsfilter::new
    @bsfilter3.setup($default_options + ["--jtokenizer", "chasen"])
    @bsfilter3.use_dummyfh

    @bsfilter0.run(@files)
    @bsfilter1.run(@files)
    @bsfilter2.run(@files)
    @bsfilter3.run(@files)

    assert_equal(1, @bsfilter0.count_message(/tokenizer ja body 朝顔/),  "@bsfilter0 2letters")
    assert_equal(0, @bsfilter0.count_message(/tokenizer ja body 向日葵/), "@bsfilter0 3letters")

    assert_equal(1, @bsfilter1.count_message(/tokenizer ja body 朝顔/), "@bsfilter1 2letters")
    assert_equal(1, @bsfilter1.count_message(/tokenizer ja body 向日葵/), "@bsfilter1 3letters")

    assert_equal(1, @bsfilter2.count_message(/tokenizer ja body 朝顔/), "@bsfilter2 2letters")
    assert_equal(0, @bsfilter2.count_message(/tokenizer ja body 向日葵/), "@bsfilter2 3letters")

    assert_equal(1, @bsfilter3.count_message(/tokenizer ja body 朝顔/), "@bsfilter3 2letters")
    assert_equal(1, @bsfilter3.count_message(/tokenizer ja body 向日葵/), "@bsfilter3 3letters")
  end

  def teardown
    unlink_all
  end
end

class TestGetLang < Test::Unit::TestCase
  def setup
    @bsfilter = Bsfilter::new
    @bsfilter.setup($default_options)
    @bsfilter.use_dummyfh
  end

  def test_euc
    @files = ["testcases/euc_plain_iso_2022_jp"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja euc/))
  end

  def test_sjis
    @files = ["testcases/sjis_plain_iso_2022_jp"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja sjis/))
  end

  def test_sjis_base64_iso_2022_jp
    @files = ["testcases/sjis_base64_iso_2022_jp"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja sjis/))
  end

  def test_sjis_base64_iso_2202_jp
    @files = ["testcases/sjis_base64_iso_2202_jp"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja sjis/))
  end

  def test_iso_2022_jp_plain
    @files = ["testcases/iso_2022_jp_plain"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja jis/))
  end

  def test_utf8_base64
    @files = ["testcases/utf8_base64"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja utf8/))
  end

  def test_utf8_plain
    @files = ["testcases/utf8_plain"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja utf8/))
  end

  def test_gb18030_base64_gb2312
    @files = ["testcases/gb18030_base64_gb2312"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/lang ja gb18030/))
  end

  def teardown
    unlink_prob_sdbm
  end
end


class TestJtokenizer < Test::Unit::TestCase
  def setup
    @files = ["testcases/iso_2022_jp_plain"]
    @bsfilter = Bsfilter::new
  end

  def test_bigram
    @bsfilter.setup($default_options + ["--jtokenizer", "bigram"])
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/tokenizer ja body 朝顔/), "2 letters")
    assert_equal(0, @bsfilter.count_message(/tokenizer ja body 向日葵/), "3 letters")
  end

  def test_mecab
    assert_nothing_raised('Warning: ignore this test if MeCab is NOT installed') do
      @bsfilter.setup($default_options + ["--jtokenizer", "mecab"])
    end
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/tokenizer ja body 朝顔/), "2 letters")
    assert_equal(1, @bsfilter.count_message(/tokenizer ja body 向日葵/), "3 letters")
  end

  def test_chasen
    assert_nothing_raised('Warning: ignore this test if chasen is NOT installed') do
      @bsfilter.setup($default_options + ["--jtokenizer", "chasen"])
    end
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/tokenizer ja body 朝顔/), "2 letters")
    assert_equal(1, @bsfilter.count_message(/tokenizer ja body 向日葵/), "3 letters")
  end

  def teardown
    unlink_all
  end
end


class TestPlainTextParser < Test::Unit::TestCase
  def setup
    @bsfilter = Bsfilter::new
    @bsfilter.setup($default_options)
    @bsfilter.use_dummyfh
  end

  def test_folding
    @files = ["testcases/folding"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/tokenizer ja body 朝顔/), "japanese")
    assert_equal(0, @bsfilter.count_message(/headtail/), "english")
  end

  def test_iso_8895_1
    @files = ["testcases/iso_8895_1_plain"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/tokenizer subject replIca/), "replIca")
    assert_equal(1, @bsfilter.count_message(/tokenizer.*elegant/), "elegant")
  end

  def teardown
    unlink_prob_sdbm
  end
end


class TestDBM < Test::Unit::TestCase
  def setup
    unlink_all
    @files = ["testcases/iso_2022_jp_plain", "testcases/ascii_plain"]
    @bsfilter = Bsfilter::new
  end

  def test_default_dbm
    @bsfilter.setup($default_options + ["-c"])
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert(File::readable?("ja.clean.sdbm.dir"), "ja.clean.sdbm.dir")
    assert(File::readable?("C.clean.sdbm.dir"), "C.clean.sdbm.dir")

    @bsfilter.setup($default_options + ["-u"])
    @bsfilter.use_dummyfh
    @bsfilter.run([])
    assert(File::readable?("ja.prob.sdbm.dir"), "ja.prob.sdbm.dir")
    assert(File::readable?("C.prob.sdbm.dir"), "C.prob.sdbm.dir")

    unlink_token_sdbm
    unlink_prob_sdbm
  end

  def test_sdbm
    @bsfilter.setup($default_options + ["--db", "sdbm", "-c"])
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert(File::readable?("ja.clean.sdbm.dir"), "ja.clean.sdbm.dir")
    assert(File::readable?("C.clean.sdbm.dir"), "C.clean.sdbm.dir")

    @bsfilter.setup($default_options + ["--db", "sdbm", "-u"])
    @bsfilter.use_dummyfh
    @bsfilter.run([])
    assert(File::readable?("ja.prob.sdbm.dir"), "ja.prob.sdbm.dir")
    assert(File::readable?("C.prob.sdbm.dir"), "C.prob.sdbm.dir")

    unlink_token_sdbm
    unlink_prob_sdbm
  end

  def test_gdbm
    assert_nothing_raised('Warning: ignore this test if GDBM is NOT installed') do
      @bsfilter.setup($default_options + ["--db", "gdbm", "-c"])
    end
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert(File::readable?("ja.clean.gdbm"), "ja.clean.gdbm")
    assert(File::readable?("C.clean.gdbm"), "C.clean.gdbm")

    @bsfilter.setup($default_options + ["--db", "gdbm", "-u"])
    @bsfilter.use_dummyfh
    @bsfilter.run([])
    assert(File::readable?("ja.prob.gdbm"), "ja.prob.gdbm")
    assert(File::readable?("C.prob.gdbm"), "C.prob.gdbm")

    unlink_token_gdbm
    unlink_prob_gdbm
  end

  def test_bdb1
    assert_nothing_raised('Warning: ignore this test if BDB1 is NOT installed') do
      @bsfilter.setup($default_options + ["--db", "bdb1", "-c"])
    end
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert(File::readable?("ja.clean.bdb1"), "ja.clean.bdb1")
    assert(File::readable?("C.clean.bdb1"), "C.clean.bdb1")

    @bsfilter.setup($default_options + ["--db", "bdb1", "-u"])
    @bsfilter.use_dummyfh
    @bsfilter.run([])
    assert(File::readable?("ja.prob.bdb1"), "ja.prob.bdb1")
    assert(File::readable?("C.prob.bdb1"), "C.prob.bdb1")

    unlink_token_bdb1
    unlink_prob_bdb1
  end

  def test_bdb
    assert_nothing_raised('Warning: ignore this test if BDB is NOT installed') do
      @bsfilter.setup($default_options + ["--db", "bdb", "-c"])
    end
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert(File::readable?("ja.clean.bdb"), "ja.clean.bdb")
    assert(File::readable?("C.clean.bdb"), "C.clean.bdb")

    @bsfilter.setup($default_options + ["--db", "bdb", "-u"])
    @bsfilter.use_dummyfh
    @bsfilter.run([])
    assert(File::readable?("ja.prob.bdb"), "ja.prob.bdb")
    assert(File::readable?("C.prob.bdb"), "C.prob.bdb")

    unlink_token_bdb
    unlink_prob_bdb
  end

  def test_qdbm
    assert_nothing_raised('Warning: ignore this test if QDBM is NOT installed') do
      @bsfilter.setup($default_options + ["--db", "qdbm", "-c"])
    end
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert(File::readable?("ja.clean.qdbm"), "ja.clean.qdbm")
    assert(File::readable?("C.clean.qdbm"), "C.clean.qdbm")

    @bsfilter.setup($default_options + ["--db", "qdbm", "-u"])
    @bsfilter.use_dummyfh
    @bsfilter.run([])
    assert(File::readable?("ja.prob.qdbm"), "ja.prob.qdbm")
    assert(File::readable?("C.prob.qdbm"), "C.prob.qdbm")

    unlink_token_qdbm
    unlink_prob_qdbm
  end
end

class TestMbox < Test::Unit::TestCase
  def setup
    @files = ["testcases/mbox"]
    @bsfilter = Bsfilter::new
  end

  def test_with_mbox
    @bsfilter.setup($default_options + ["--mbox"])
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(3, @bsfilter.count_message(/^file/))
  end

  def test_without_mbox
    @bsfilter.setup($default_options)
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/file/))
  end

  def teardown
    unlink_prob_sdbm
  end
end

class TestHeaderParser < Test::Unit::TestCase
  def setup
    @bsfilter = Bsfilter::new
    @bsfilter.setup($default_options)
    @bsfilter.use_dummyfh
  end

  def test_header_parser
    @files = ["testcases/header"]
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/^tokenizer received host1/), "^tokenizer received host1")
    assert_equal(1, @bsfilter.count_message(/^tokenizer received host2/), "^tokenizer received host2")
    assert_equal(0, @bsfilter.count_message(/^tokenizer received host3/), "^tokenizer received host3") # drop 2nd hop
    assert_equal(0, @bsfilter.count_message(/abcdefgh/), "abcdefgh") # drop ID
    assert_equal(0, @bsfilter.count_message(/signaturebody/), "signaturebody") # drop signature
  end

  def teardown
    unlink_prob_sdbm
  end
end

class TestEOL < Test::Unit::TestCase
  def setup
    @bsfilter = Bsfilter::new
    @bsfilter.setup($default_options + ["--pipe", "--insert-revision"])
    @bsfilter.use_dummyfh
  end

  def test_lf
    @files = ["testcases/lf"]
    @bsfilter.run(@files)

    assert_equal(11, @bsfilter.count_pipe(/\n/), '\n')
    assert_equal(0, @bsfilter.count_pipe(/\r/), '\r')
    assert_equal(0, @bsfilter.count_pipe(/\r\n/), '\r\n')
  end

  def test_cr
    @files = ["testcases/cr"]
    @bsfilter.run(@files)

    assert_equal(0, @bsfilter.count_pipe(/\n/), '\n')
    assert_equal(11, @bsfilter.count_pipe(/\r/), '\r')
    assert_equal(0, @bsfilter.count_pipe(/\r\n/), '\r\n')
  end

  def test_crlf
    @files = ["testcases/crlf"]
    @bsfilter.run(@files)

    assert_equal(11, @bsfilter.count_pipe(/\n/), '\n')
    assert_equal(11, @bsfilter.count_pipe(/\r/), '\r')
    assert_equal(11, @bsfilter.count_pipe(/\r\n/), '\r\n')
  end
end


class TestHtmlParser < Test::Unit::TestCase
  def setup
    @files = ["testcases/html"]
    @bsfilter = Bsfilter::new
    @bsfilter.setup($default_options + ["-c"])
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)

    @bsfilter.setup($default_options + ["-u"])
    @bsfilter.use_dummyfh
    @bsfilter.run([])
  end

  def test_default
    @bsfilter.setup($default_options)
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(1, @bsfilter.count_message(/^word.*plain_text_part/), "^word.*plain_text_part")
    assert_equal(1, @bsfilter.count_message(/^tokenizer.*ABCDEF/), "ABCDEF")
    assert_equal(1, @bsfilter.count_message(/^tokenizer.*after_atag/), "after_atag")

    assert_equal(0, @bsfilter.count_message(/^tokenizer.*after_html/), "after_html")
    assert_equal(0, @bsfilter.count_message(/^tokenizer.*after_body/), "after_body")
    assert_equal(0, @bsfilter.count_message(/^tokenizer.*fontsize0/), "fontsize0")
    assert_equal(0, @bsfilter.count_message(/^tokenizer.*fontsize1/), "fontsize1")
    assert_equal(0, @bsfilter.count_message(/^tokenizer.*displaynone/), "displaynone")
    assert_equal(1, @bsfilter.count_message(/^tokenizer url 192\.168\.0\.1/), "192.168.0.1")
    assert_equal(1, @bsfilter.count_message(/^tokenizer url www/), "www")
  end

  def test_ignore_plain_text_part
    @bsfilter.setup($default_options + ["--ignore-plain-text-part"])
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(0, @bsfilter.count_message(/^word.*plain_text_part/))
  end

  def test_ignore_after_last_atag
    @bsfilter.setup($default_options + ["--ignore-after-last-atag"])
    @bsfilter.use_dummyfh
    @bsfilter.run(@files)
    assert_equal(0, @bsfilter.count_message(/^tokenizer.*after_atag/))
  end

  def teardown
    unlink_all
  end
end

class TestTokenizerOptionCombination < Test::Unit::TestCase
  def setup
    @files = Dir.glob("testcases/[^C]*")
    @option_elements = ["--disable-utf-8", "--ignore-header", "--ignore-body", "--disable-degeneration",
                        "--mark-in-token @", "--ignore-plain-text-part", "--ignore-after-last-atag"]
  end

  def test_all
    i = 0
    imax = 2 ** @option_elements.length

    while (i < imax)
      j = 0
      option_array = Array::new
      while (j < @option_elements.length)
        if ((i >> j) % 2 == 1)
          option_array.concat(@option_elements[j].split)
        end
        j += 1
      end
      bsfilter = Bsfilter::new
      bsfilter.setup($default_options + option_array + ["-q"])
      bsfilter.run(@files)
      i += 1
    end
  end

  def teardown
    unlink_prob_sdbm
  end
end
