# mktab.sh, 17 Feb 05
#
# Extract data from various files (generated by ngram)
# needed for C construct frequency tables

source ../config.files

LOGICAL_OPS=" && || "
SHIFT_OPS=" >> << "
BITWISE_OPS=" & | ^ "
BIT_OPS=" ~ ^= <<= >>= &= |= $SHIFT_OPS $BITWISE_OPS"
MULT_OPS=" * / % "
ADD_OPS=" - + "
ASSIGN_OPS=" = += -= *= /= %= |= &= ^= <<= >>= "
ARITH_OPS=" += -= *= /= %= $MULT_OPS $ADD_OPS"
REL_OPS=" < <= > >= "
EQUAL_OPS=" == != "
POSTFIX_OPS=" . -> v++ v-- "				; export POSTFIX_OPS
PREFIX_OPS=" ++v --v "					; export PREFIX_OPS
UNARY_OPS=" SIZEOF &v *v +v -v ~ ! "			; export UNARY_OPS
TERNARY_OPS=" ? ?: "
RVALUE_OPS="$BIT_OPS $LOGICAL_OPS $ARITH_OPS $REL_OPS $EQUAL_OPS $UNARY_OPS"  ; export RVALUE_OPS
EXPR_OPS="$RVALUE_OPS $TERNARY_OPS $ASSIGN_OPS"		; export EXPR_OPS
STORE_CLASS=" TYPEDEF EXTERN STATIC REGISTER "
TYPE_SPEC="  VOID CHAR SHORT INT LONG FLOAT DOUBLE SIGNED UNSIGNED STRUCT UNION ENUM TYPEDEF-NAME "
TYPE_QUAL=" CONST VOLATILE RESTRICT "

echo "Extracting TABLE data to $LOC_TAB_DIR"

awk -f cchar_op.awk < $C_CNT_FILE > $GLOB_TAB_DIR/cchar_op
awk -v D_FILE="ccharlen.d" -v ADD_COMMA="Y" -f $EXTRTWOCOLS < $C_CNT_FILE | $MKTWOCOL > $GLOB_TAB_DIR/cchar_len
awk -v D_FILE="hex_ndigit.d" -v ADD_COMMA="Y" -f $EXTRTWOCOLS < $C_CNT_FILE | $MKFOURCOL > $GLOB_TAB_DIR/hex_num_dig

if [ $GEN_DATA == 1 ]; then
   awk -v D_FILE="intsuffix.d" -f $EXTRTWOCOLS < $H_CNT_FILE > $LOC_TAB_DIR/int_suffix.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/int_suffix.h
   awk -v D_FILE="intsuffix.d" -f $EXTRTWOCOLS < $C_CNT_FILE > $LOC_TAB_DIR/int_suffix.c
   fi
cat $LOC_TAB_DIR/int_suffix.h $LOC_TAB_DIR/int_suffix.c | $MERGETWOCOL | $MKTWOCOL > $GLOB_TAB_DIR/int_suffix

if [ $GEN_DATA == 1 ]; then
   awk -v D_FILE="int_cons_kind.d" -f $EXTRTWOCOLS < $H_CNT_FILE > $LOC_TAB_DIR/int_cons_kind.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/int_cons_kind.h
   awk -v D_FILE="int_cons_kind.d" -f $EXTRTWOCOLS < $C_CNT_FILE > $LOC_TAB_DIR/int_cons_kind.c
   fi
cat $LOC_TAB_DIR/int_cons_kind.h $LOC_TAB_DIR/int_cons_kind.c | $MERGETWOCOL  > $GLOB_TAB_DIR/int_cons_kind

awk -v D_FILE="int_cons_type.d" -f $EXTRTABLE < $C_CNT_FILE > $GLOB_TAB_DIR/cons_int_type

if [ $GEN_DATA == 1 ]; then
   awk -v D_FILE="fltsuffix.d" -f $EXTRTWOCOLS < $H_CNT_FILE > $LOC_TAB_DIR/flt_suffix.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/flt_suffix.h
   awk -v D_FILE="fltsuffix.d" -f $EXTRTWOCOLS < $C_CNT_FILE > $LOC_TAB_DIR/flt_suffix.c
   fi
cat $LOC_TAB_DIR/flt_suffix.h $LOC_TAB_DIR/flt_suffix.c | $MERGETWOCOL  > $GLOB_TAB_DIR/flt_suffix

if [ $GEN_DATA == 1 ]; then
   awk -f keyword.awk < $H_CNT_FILE > $LOC_TAB_DIR/key_freq.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/key_freq.h
   awk -f keyword.awk < $C_CNT_FILE > $LOC_TAB_DIR/key_freq.c
   fi
cat $LOC_TAB_DIR/key_freq.h $LOC_TAB_DIR/key_freq.c | awk -f mergekeyfreq.awk  | $MKTWOCOL > $GLOB_TAB_DIR/key_freq

echo "bigrams"

awk -v ANY_TOK=" CHAR_CONST " -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/cchar_bigram
awk -v ANY_TOK=" FLOAT_CONST " -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/flt_cons_bigram
awk -v ANY_TOK="$POSTFIX_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/postfix_op_tok
awk -v FIRST_TOK="$UNARY_OPS $PREFIX_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/unary_op_tok
awk -v ANY_TOK="$MULT_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE > $GLOB_TAB_DIR/mult_op_tok
awk -v ANY_TOK="$ADD_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/add_op_tok
awk -v ANY_TOK="$SHIFT_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/shift_op_tok
awk -v ANY_TOK="$REL_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/rel_op_tok
awk -v ANY_TOK="$EQUAL_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/equal_op_tok
awk -v ANY_TOK="$BITWISE_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/bitwise_op_tok
awk -v ANY_TOK="$LOGICAL_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/logical_op_tok
awk -v ANY_TOK="$MULT_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/mult_op_tok
awk -v ANY_TOK="$TERNARY_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/ternary_op_tok
awk -v ANY_TOK="$ASSIGN_OPS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/assign_op_tok
awk -v FIRST_TOK="$STORE_CLASS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/storclass_bigram
awk -v FIRST_TOK="$STORE_CLASS" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $H_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/storclass_bigram_h
awk -v ANY_TOK="$TYPE_SPEC" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/typespec_bigram
awk -v ANY_TOK="$TYPE_QUAL" -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/typequal_bigram
awk -v SECOND_TOK=" CASE DEFAULT " -v NOT_TOK=" SPACE LINE_START " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL > $GLOB_TAB_DIR/case_bigram

awk -v ANY_TOK=" INT_CONST " -v NOT_TOK=" SPACE LINE_START " -v PRIM_TOK=" INT_CONST " -f bigram.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/int_cons_bigram

if [ $GEN_DATA == 1 ]; then
   awk -v FIRST_TOK=" ENUM STRUCT UNION " -v SECOND_TOK=" IDENT " -f bigram.awk < $H_CNT_FILE > $LOC_TAB_DIR/esu_ident_cnt.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/esu_ident_cnt.h
   awk -v FIRST_TOK=" ENUM STRUCT UNION " -v SECOND_TOK=" IDENT " -f bigram.awk < $C_CNT_FILE > $LOC_TAB_DIR/esu_ident_cnt.c
   fi
cat $LOC_TAB_DIR/esu_ident_cnt.h $LOC_TAB_DIR/esu_ident_cnt.c | awk -f mergeesu.awk  > $GLOB_TAB_DIR/esu_ident_cnt

if [ $GEN_DATA == 1 ]; then
   awk -v FIRST_TOK=" -> \. #DEFINE STRUCT CASE GOTO ENUM UNION " -v SECOND_TOK=" IDENT " -f bigram.awk < $H_CNT_FILE > $LOC_TAB_DIR/ns_tok_ident.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/ns_tok_ident.h
   awk -v FIRST_TOK=" -> \. #DEFINE STRUCT CASE GOTO ENUM UNION " -v SECOND_TOK=" IDENT " -f bigram.awk < $C_CNT_FILE > $LOC_TAB_DIR/ns_tok_ident.c
   fi
cat $LOC_TAB_DIR/ns_tok_ident.h $LOC_TAB_DIR/ns_tok_ident.c | awk -f mergefirstcol.awk | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/ns_tok_ident

echo "misc"
awk -f charfreq.awk < $C_CNT_FILE | $MKFOURCOL > $GLOB_TAB_DIR/char_freq
if [ $GEN_DATA == 1 ]; then
   awk -f sptoksp.awk < $H_CNT_FILE > $LOC_TAB_DIR/sptoksp.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/sptoksp.h
   awk -f sptoksp.awk < $C_CNT_FILE > $LOC_TAB_DIR/sptoksp.c
   fi
cat $LOC_TAB_DIR/sptoksp.h $LOC_TAB_DIR/sptoksp.c | awk -f mergesptoksp.awk  > $GLOB_TAB_DIR/sptoksp
if [ $GEN_DATA == 1 ]; then
   awk -f constant.awk < $H_CNT_FILE > $LOC_TAB_DIR/constant.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/constant.h
   awk -f constant.awk < $C_CNT_FILE > $LOC_TAB_DIR/constant.c
   fi
cat $LOC_TAB_DIR/constant.h $LOC_TAB_DIR/constant.c | $MERGETWOCOL  > $GLOB_TAB_DIR/constant
awk -f conskind.awk < $C_CNT_FILE > $GLOB_TAB_DIR/cons_kind
awk -f escseq.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/escape_seq
awk -f charcons.awk < $C_CNT_FILE > $GLOB_TAB_DIR/char_cons_kind
if [ $GEN_DATA == 1 ]; then
   echo "Getting string literals"
   getstrlit.sh
   fi
strlitinfo.sh | awk -f strlitinfo.awk > $GLOB_TAB_DIR/str_lit_info
awk -f punctfreq.awk < $C_CNT_FILE | $SORTLASTCOL | $MKFOURCOL > $GLOB_TAB_DIR/punct_freq
if [ $GEN_DATA == 1 ]; then
   awk -f commentinfo.awk < $H_CNT_FILE > $LOC_TAB_DIR/comment_info.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/comment_info.h
   awk -f commentinfo.awk < $C_CNT_FILE > $LOC_TAB_DIR/comment_info.c
   fi
cat $LOC_TAB_DIR/comment_info.h $LOC_TAB_DIR/comment_info.c | awk -f mergelastcol.awk  > $GLOB_TAB_DIR/comment_info
awk -f tokeneol.awk < $C_CNT_FILE | $MKTWOCOL > $GLOB_TAB_DIR/token_eol
awk -f soltoken.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/sol_token
lognotcont.sh | $SORTLASTCOL > $GLOB_TAB_DIR/log_not_context
andcont.sh | awk -f jointworows.awk | $SORTLASTCOL > $GLOB_TAB_DIR/and_context
orcont.sh | awk -f jointworows.awk | $SORTLASTCOL > $GLOB_TAB_DIR/or_context
relcont.sh | awk -f joinfourrows.awk | $SORTLASTCOL > $GLOB_TAB_DIR/rel_context
awk -f typespecseq.awk < $C_CNT_FILE | $SORTLASTCOL | $MKTWOCOL > $GLOB_TAB_DIR/typespec_seq

if [ $GEN_DATA == 1 ]; then
   awk -f charcnt.awk < $H_CNT_FILE > $LOC_TAB_DIR/char_cnt.h
   echo "END_DOT_H" >> $LOC_TAB_DIR/char_cnt.h
   awk -f charcnt.awk < $C_CNT_FILE > $LOC_TAB_DIR/char_cnt.c
   fi
cat $LOC_TAB_DIR/char_cnt.h $LOC_TAB_DIR/char_cnt.c | awk -f mergecol3.awk  > $GLOB_TAB_DIR/char_cnt

awk -f jumpcurl.awk < $C_CNT_FILE > $GLOB_TAB_DIR/jump_curl


rm -f $TEMP_FILE
