comparison cjk_impl.c @ 2:754a4550c64e

- added arabic, greek, hebrew and turkish DFAs - new UCS-2LE/BE DFAs - now arabic_impl.c uses arabic DFAs - dfa common macros have been moved to dfa.h - minor cleanups

1:04f2be1c8464

 * This code is derivative of guess.c of Gauche-0.8.3.
 * The following is the original copyright notice.
 */

/*
 * guess.c - guessing character encoding 
 *
 *   Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
 * 
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 * 
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the

 *   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "libguess.h"

/* take precedence if scores are same. you can customize the order as: */
/* ORDER_** &highest, &second, ... &lowest */
#define ORDER_JP &utf8, &sjis, &eucj
#define ORDER_TW &utf8, &big5
#define ORDER_CN &utf8, &gb2312, &gb18030
#define ORDER_KR &utf8, &euck, &johab

/* workaround for that glib's g_convert can't convert
   properly from UCS-2BE/LE trailing after BOM. */
#define WITH_G_CONVERT 1
/* #undef WITH_G_CONVERT */

#else
const char UCS_2BE[] = "UCS-2BE";
const char UCS_2LE[] = "UCS-2LE";
#endif

/* data types */
typedef struct guess_arc_rec
{
    unsigned int next;          /* next state */
    double score;               /* score */
} guess_arc;

typedef struct guess_dfa_rec
{
    signed char (*states)[256];
    guess_arc *arcs;
    int state;
    double score;
} guess_dfa;

/* macros */
#define DFA_INIT(st, ar) \
    { st, ar, 0, 1.0 }

#define DFA_NEXT(dfa, ch)                               \
    do {                                                \
        int arc__;                                      \
        if (dfa.state >= 0) {                           \
            arc__ = dfa.states[dfa.state][ch];          \
            if (arc__ < 0) {                            \
                dfa.state = -1;                         \
            } else {                                    \
                dfa.state = dfa.arcs[arc__].next;       \
                dfa.score *= dfa.arcs[arc__].score;     \
            }                                           \
        }                                               \
    } while (0)

#define DFA_ALIVE(dfa)  (dfa.state >= 0)

/* include DFA table generated by guess.scm */
#include "guess_tab.c"

    DFA_NEXT(utf8, '\0'); //Bug #53

    if(DFA_ALIVE(utf8))
        return 1;
    else 
        return 0;
}

const char *guess_jp(const char *buf, int buflen)
{

2:754a4550c64e

 * This code is derivative of guess.c of Gauche-0.8.3.
 * The following is the original copyright notice.
 */

/*
 * guess.c - guessing character encoding
 *
 *   Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the

 *   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "libguess.h"
#include "dfa.h"

/* workaround for that glib's g_convert can't convert
   properly from UCS-2BE/LE trailing after BOM. */
#define WITH_G_CONVERT 1
/* #undef WITH_G_CONVERT */

#else
const char UCS_2BE[] = "UCS-2BE";
const char UCS_2LE[] = "UCS-2LE";
#endif

/* take precedence if scores are same. you can customize the order as: */
/* ORDER_** &highest, &second, ... &lowest */
#define ORDER_JP &utf8, &sjis, &eucj
#define ORDER_TW &utf8, &big5
#define ORDER_CN &utf8, &gb2312, &gb18030
#define ORDER_KR &utf8, &euck, &johab

/* include DFA table generated by guess.scm */
#include "guess_tab.c"

    DFA_NEXT(utf8, '\0'); //Bug #53

    if(DFA_ALIVE(utf8))
        return 1;
    else
        return 0;
}

const char *guess_jp(const char *buf, int buflen)
{