How to enable CJV searching in 1.4.6? #176

dwong33 · 2024-08-29T06:14:36Z

dwong33
Aug 29, 2024

I want to enable the CJV search in 1.4.6, but when referring to the FAQs, the tutorial tell me to enable that under sphinx.conf.

But I do not find sphinx.conf in /etc/piler, only manticore.conf, and it looks something like this. How should we configure if we want to enable CJV search? Thank you.

#!/usr/bin/php
<?php

define('LOCALSTATEDIR', '/var');
define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");

# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
# The following settings contains English and some Latin extras
define('CHARSET_TABLE', "0..9, english, _, \
                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");

define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');

define('RT', 1);

jsuto · 2024-08-29T13:09:22Z

jsuto
Aug 29, 2024
Maintainer

Did you mean CJK support? See https://manual.manticoresearch.com/Creating_a_table/NLP_and_tokenization/CJK?client=CONFIG#Chinese,-Japanese-and-Korean-(CJK)-languages for details how to enable it.

Also be sure to set enable_cjk=1 in piler.conf

1 reply

dwong33 Aug 30, 2024
Author

Thank you for your reply, unfortunately, I cannot get it working.

My current manticore.conf is:

index piler1
{
    type = rt
    path = /var/piler/manticore/piler1
    rt_mem_limit = 512M
    stored_fields =
    min_word_len = 1
    min_prefix_len = 5
    morphology = icu_chinese
    charset_table  = 0..9, cjk, english, _, \
                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171
    # See https://manual.manticoresearch.com/Creating_an_index/Data_types#Row-wise-and-columnar-attribute-storages
    # if you want to enable columnar storage
    # columnar_attrs = *
    rt_field = sender
    rt_field = rcpt
    rt_field = senderdomain
    rt_field = rcptdomain
    rt_field = subject
    rt_field = body
    rt_field = attachment_types
    rt_attr_bigint = arrived
    rt_attr_bigint = sent
    rt_attr_uint = size
    rt_attr_uint = direction
    rt_attr_uint = folder
    rt_attr_uint = attachments
}

index tag1
{
    type = rt
    path = /var/piler/manticore/tag1
    rt_mem_limit = 16M
    stored_fields = tag
    min_word_len = 2
    min_prefix_len = 5
    morphology = icu_chinese
    charset_table  = 0..9, cjk, english, _, \
                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171
    rt_field = tag
    rt_attr_bigint = mid
    rt_attr_uint = uid
}

index note1
{
    type = rt
    path = /var/piler/manticore/note1
    rt_mem_limit = 16M
    stored_fields = note
    min_word_len = 2
    min_prefix_len = 5
    morphology = icu_chinese
    charset_table  = 0..9, cjk, english, _, \
                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171
    rt_field = note
    rt_attr_bigint = mid
    rt_attr_uint = uid
}


index audit1
{
    type = rt
    path = /var/piler/manticore/audit1
    rt_mem_limit = 16M
    stored_fields = *
    min_word_len = 2
    min_prefix_len = 5
    morphology = icu_chinese
    charset_table  = 0..9, cjk, english, _, \
                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171
    rt_field = email
    rt_field = ipaddr
    rt_field = description
    rt_attr_bigint = ts
    rt_attr_bigint = meta_id
    rt_attr_uint = action
}

searchd
{
        listen                  = 127.0.0.1:9312
        listen                  = 127.0.0.1:9306:mysql
        listen                  = 127.0.0.1:9307:mysql_readonly
        log                     = /var/piler/manticore/manticore.log
        binlog_max_log_size     = 256M
        binlog_path             = /var/piler/manticore
        binlog_flush            = 2
        query_log               = /var/piler/manticore/query.log
        network_timeout         = 5
        pid_file                = /var/run/piler/searchd.pid
        seamless_rotate         = 1
        preopen_tables          = 1
        unlink_old              = 1
        thread_stack            = 512k
        rt_flush_period         = 300

}

And I have set enable_cjk=1 in piler.conf. When I perform some searching, it returns no result, the mail.log returns something like this:

sphinx query: 'SELECT id FROM piler1 WHERE        MATCH('(  電郵 ) &  ( (@sender exampleXexampleXcom) | (@rcpt exampleXexampleXcom) ) ') ORDER BY `sent` DESC LIMIT 0,50 OPTION max_matches=1000' in 0.00 s, 50 hits, 4490 total found
2024-08-30T03:56:04.455938-04:00 ce08a4742eeb piler-webui[33]: sphinx query: 'SELECT mid, tag FROM tag1 WHERE uid=2 AND mid IN (59992,59991,59990,59989,59988,59987,59986,59985,59984,59983,59770,59769,59768,59767,59766,59765,59764,59763,59762,59761,59760,59759,59758,59757,59756,59755,59754,59753,59752,59751,59750,59749,59748,59747,59746,59745,59744,59743,59741,59740,59739,59737,59738,59736,59735,59734,59733,59732,59731,59730)' in 0.00 s, 0 hits, 0 total found
2024-08-30T03:56:04.456600-04:00 ce08a4742eeb piler-webui[33]: sphinx query: 'SELECT mid, note FROM note1 WHERE uid=2 AND mid IN (59992,59991,59990,59989,59988,59987,59986,59985,59984,59983,59770,59769,59768,59767,59766,59765,59764,59763,59762,59761,59760,59759,59758,59757,59756,59755,59754,59753,59752,59751,59750,59749,59748,59747,59746,59745,59744,59743,59741,59740,59739,59737,59738,59736,59735,59734,59733,59732,59731,59730)' in 0.00 s, 0 hits, 0 total found

jsuto · 2024-08-30T08:09:29Z

jsuto
Aug 30, 2024
Maintainer

You have 4490 hits, and you should see the last 50 of it.

Try the following mysql query in the piler database:

'SELECT id, piler_id FROM metadata WHERE id IN (59992,59991,59990,59989,59988,59987,59986,59985,59984,59983,59770,59769,59768,59767,59766,59765,59764,59763,59762,59761,59760,59759,59758,59757,59756,59755,59754,59753,59752,59751,59750,59749,59748,59747,59746,59745,59744,59743,59741,59740,59739,59737,59738,59736,59735,59734,59733,59732,59731,59730);

1 reply

dwong33 Aug 30, 2024
Author

After executing the query, it returns the most recent emails, these does not include the phrase I want to search.

I tried to reindex all the emails. And after that, when I try to search the terms, it returns no entry...

2024-08-30T04:42:37.663798-04:00 ce08a4742eeb piler-webui[33]: sphinx query: 'SELECT id FROM piler1 WHERE        MATCH('郵件') ORDER BY `sent` DESC LIMIT 0,250 OPTION max_matches=1000' in 0.00 s, 0 hits, 0 total found
2024-08-30T04:44:15.140306-04:00 ce08a4742eeb piler-webui[32]: sphinx query: 'INSERT INTO audit1 (ts, email, action, ipaddr, meta_id, description) VALUES(?,?,?,?,?,?)' in 0.00 s, 0 hits, 0 total found
2024-08-30T04:44:15.140793-04:00 ce08a4742eeb piler-webui[32]: sphinx query: 'SELECT id FROM piler1 WHERE        MATCH(' 日') ORDER BY `sent` DESC LIMIT 0,250 OPTION max_matches=1000' in 0.00 s, 0 hits, 0 total found
2024-08-30T04:44:16.406153-04:00 ce08a4742eeb piler-webui[33]: sphinx query: 'INSERT INTO audit1 (ts, email, action, ipaddr, meta_id, description) VALUES(?,?,?,?,?,?)' in 0.00 s, 0 hits, 0 total found
2024-08-30T04:44:16.406643-04:00 ce08a4742eeb piler-webui[33]: sphinx query: 'SELECT id FROM piler1 WHERE        MATCH(' 日') ORDER BY `sent` DESC LIMIT 0,250 OPTION max_matches=1000' in 0.00 s, 0 hits, 0 total found

jsuto · 2024-09-04T05:11:38Z

jsuto
Sep 4, 2024
Maintainer

Can you attach 2-3 non-sensitive emails with full headers? I'd test it on my dev environment. Feel free to obfuscate the headers in order that they contain no sensitive info, but still having valid entries.

1 reply

dwong33 Sep 4, 2024
Author

[Edit] Email Deleted
Hi I attached three email samples here, please check. Thank you very much! I will delete the email once you have downloaded them.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

How to enable CJV searching in 1.4.6? #176

{{title}}

{{editor}}'s edit

{{editor}}'s edit

Replies: 3 comments 3 replies

{{title}}

{{editor}}'s edit

{{editor}}'s edit

{{title}}

{{title}}

{{title}}

{{title}}

{{title}}

{{editor}}'s edit

{{editor}}'s edit

Select a reply

How to enable CJV searching in 1.4.6? #176

dwong33 Aug 29, 2024

Replies: 3 comments · 3 replies

jsuto Aug 29, 2024 Maintainer

dwong33 Aug 30, 2024 Author

jsuto Aug 30, 2024 Maintainer

dwong33 Aug 30, 2024 Author

jsuto Sep 4, 2024 Maintainer

dwong33 Sep 4, 2024 Author

dwong33
Aug 29, 2024

Replies: 3 comments 3 replies

jsuto
Aug 29, 2024
Maintainer

dwong33 Aug 30, 2024
Author

jsuto
Aug 30, 2024
Maintainer

dwong33 Aug 30, 2024
Author

jsuto
Sep 4, 2024
Maintainer

dwong33 Sep 4, 2024
Author