annotate cjk_impl.c @ 3:70e2c306231e

- implemented dfa utility functions. - added dfa.c. - rewrote guess functions for ar, gr, hw and tr scripts with dfa utilities. - guess functions for cjk scripts too.

/*
 * This code is derivative of guess.c of Gauche-0.8.3.
 * The following is the original copyright notice.
 */

/*
 * guess.c - guessing character encoding
 *
 *   Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 *   3. Neither the name of the authors nor the names of its contributors
 *      may be used to endorse or promote products derived from this
 *      software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 *   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 *   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 *   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 *   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 *   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "libguess.h"
#include "dfa.h"

#include <stdio.h>

/* take precedence if scores are same. you can customize the order as: */
/* ORDER_** &highest, &second, ... &lowest */
#define ORDER_JP &utf8, &sjis, &eucj
#define ORDER_TW &utf8, &big5
#define ORDER_CN &utf8, &gb2312, &gb18030
#define ORDER_KR &utf8, &euck, &johab

/* include DFA table generated by guess.scm */
#include "guess_tab.c"

int dfa_validate_utf8(const char *buf, int buflen)
{
    int i;
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8");

    for (i = 0; i < buflen; i++) {
        int c = (unsigned char) buf[i];

        if (DFA_ALIVE(utf8))
            DFA_NEXT(utf8, c);
        else
            break;
    }

    DFA_NEXT(utf8, '\0'); //Bug #53

    if(DFA_ALIVE(utf8))
        return 1;
    else
        return 0;
}

const char *guess_jp(const char *buf, int buflen)
{
    int i;
    const char *rv = NULL;
    guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar, "EUC-JP");
    guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar, "SJIS");
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8");
    guess_dfa *top = NULL;

    guess_dfa *order[] = { ORDER_JP, NULL };

    for (i = 0; i < buflen; i++) {
        int c = (unsigned char) buf[i];

        /* special treatment of iso-2022 escape sequence */
        if (c == 0x1b) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[++i];
                if (c == '$' || c == '(')
                    return "ISO-2022-JP";
            }
        }

        /* special treatment of BOM */
        if (i == 0 && c == 0xff) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xfe)
                    return UCS_2LE;
            }
        }
        if (i == 0 && c == 0xfe) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xff)
                    return UCS_2BE;
            }
        }

        rv = dfa_process(order, c);
        if(rv)
            return rv;

        if (dfa_none(order)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    top = dfa_top(order);
    if(top)
        return top->name;
    else
        return NULL;
}

const char *guess_tw(const char *buf, int buflen)
{
    int i;
    const char *rv = NULL;
    guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar, "BIG5");
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8");
    guess_dfa *top = NULL;

    guess_dfa *order[] = { ORDER_TW, NULL };

    for (i = 0; i < buflen; i++) {
        int c = (unsigned char) buf[i];

        /* special treatment of iso-2022 escape sequence */
        if (c == 0x1b) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[++i];
                if (c == '$' || c == '(')
                    return "ISO-2022-TW";
            }
        }

        /* special treatment of BOM */
        if (i == 0 && c == 0xff) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xfe)
                    return UCS_2LE;
            }
        }
        if (i == 0 && c == 0xfe) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xff)
                    return UCS_2BE;
            }
        }

        rv = dfa_process(order, c);
        if(rv)
            return rv;

        if (dfa_none(order)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    top = dfa_top(order);
    if (top)
        return top->name;
    else
        return NULL;
}

const char *guess_cn(const char *buf, int buflen)
{
    int i;
    const char *rv = NULL;
    guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar, "GB2312");
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8");
    guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar, "GB18030");
    guess_dfa *top = NULL;

    guess_dfa *order[] = { ORDER_CN, NULL };

    for (i = 0; i < buflen; i++) {
        int c = (unsigned char) buf[i];
        int c2;

        /* special treatment of iso-2022 escape sequence */
        if (c == 0x1b) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                c2 = (unsigned char) buf[i + 2];
                if (c == '$' && (c2 == ')' || c2 == '+'))
                    return "ISO-2022-CN";
            }
        }

        /* special treatment of BOM */
        if (i == 0 && c == 0xff) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xfe)
                    return UCS_2LE;
            }
        }
        if (i == 0 && c == 0xfe) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xff)
                    return UCS_2BE;
            }
        }

        rv = dfa_process(order, c);
        if(rv)
            return rv;

        if (dfa_none(order)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    top = dfa_top(order);
    if(top)
        return top->name;
    else
        return NULL;
}

const char *guess_kr(const char *buf, int buflen)
{
    int i;
    const char *rv = NULL;
    guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar, "EUC-KR");
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar, "UTF-8");
    guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar, "JOHAB");
    guess_dfa *top = NULL;

    guess_dfa *order[] = { ORDER_KR, NULL };

    for (i = 0; i < buflen; i++) {
        int c = (unsigned char) buf[i];
        int c2;

        /* special treatment of iso-2022 escape sequence */
        if (c == 0x1b) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                c2 = (unsigned char) buf[i + 2];
                if (c == '$' && c2 == ')')
                    return "ISO-2022-KR";
            }
        }

        /* special treatment of BOM */
        if (i == 0 && c == 0xff) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xfe)
                    return UCS_2LE;
            }
        }
        if (i == 0 && c == 0xfe) {
            if (i < buflen - 1) {
                c = (unsigned char) buf[i + 1];
                if (c == 0xff)
                    return UCS_2BE;
            }
        }

        rv = dfa_process(order, c);
        if(rv)
            return rv;

        if (dfa_none(order)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    top = dfa_top(order);
    if(top)
        return top->name;
    else
        return NULL;
}