Fix reading UTF-8 literals (#592)

Author: kio-watanabe

cobj/pplex.l.m4

@@ -981,6 +981,7 @@ ppinput (char *buff, int max_size)
 	int	i;
 	int	n;
 	int	coln;
+	int buff_len = 0;
 	char	*str1 = NULL;
 	char	*str2 = NULL;
 	int comment_counter = 0;
@@ -1152,8 +1153,19 @@ start:
 		within_comment = 0;
 	}
 
+#ifdef I18N_UTF8
+	unsigned char *p = (unsigned char *)buff;
+	if(utf8_ext_pick(p)){
+		buff_len = (int) utf8_calc_sjis_size(p, strlen(buff));
+	}else{
+		buff_len = n;
+	}
+#else /*!I18N_UTF8*/
+	buff_len = n;
+#endif /*I18N_UTF8*/
+
 	/* check the text that is longer than cb_text_column */
-	if (n > cb_text_column + 1 && cb_source_format != CB_FORMAT_VARIABLE) {
+	if (buff_len > cb_text_column + 1 && cb_source_format != CB_FORMAT_VARIABLE) {
 
 		/* show warning if it is not whitespaces */
 		if (cb_warn_column_overflow && last_line_2 < cb_source_line - 1) {

cobj/scanner.l.m4

@@ -815,6 +815,12 @@ read_literal (int mark, enum cb_category category)
 	}
 
 	while ((c = input ()) != EOF) {
+#if EOF != 0
+		if (unlikely (c == 0)){ 
+			cb_error(_("The literal is not properly closed by %c."), mark);
+			break;
+		}
+#endif
 		plexbuff[i++] = c;
 		if (c == mark && (c = input ()) != mark) {
 			i--;

tests/cobol_utf8.src/pic-n.at

@@ -520,3 +520,40 @@ AT_CHECK([echo -n '゜ポンデリング' | nkf --ic=UTF-8 --oc=Shift_JIS > out2
 AT_CHECK([diff out1.txt out2.txt])
 
 AT_CLEANUP
+
+AT_SETUP([PIC N WRITE FROM Japanese literal])
+export LC_ALL=''
+
+AT_DATA([prog.cob], [
+       IDENTIFICATION   DIVISION.
+       PROGRAM-ID.      prog.
+       ENVIRONMENT      DIVISION.
+       INPUT-OUTPUT     SECTION.
+       FILE-CONTROL.
+           SELECT TEST-FILE ASSIGN TO "TESTFILE"
+                  ORGANIZATION IS SEQUENTIAL.
+       DATA DIVISION.
+       FILE SECTION.
+       FD  TEST-FILE.
+       01  TEST-DATA     PIC N(18).
+       WORKING-STORAGE SECTION.
+       PROCEDURE DIVISION.
+           OPEN  OUTPUT TEST-FILE.
+           WRITE TEST-DATA FROM "縄文弥生古墳飛鳥奈良平安鎌倉室町江戸".
+           CLOSE TEST-FILE.
+
+           OPEN INPUT TEST-FILE.
+           READ TEST-FILE
+               NOT AT END
+                   DISPLAY TEST-DATA
+           END-READ.
+           CLOSE TEST-FILE.
+])
+
+AT_CHECK([cobj prog.cob])
+AT_CHECK([java prog > out1.txt])
+AT_CHECK([echo -n '縄文弥生古墳飛鳥奈良平安鎌倉室町江戸
+' | nkf --ic=UTF-8 --oc=Shift_JIS > out2.txt])
+AT_CHECK([diff out1.txt out2.txt])
+
+AT_CLEANUP

tests/cobol_utf8.src/pic-x.at

@@ -524,6 +524,43 @@ AT_CHECK([diff out1.txt out2.txt])
 
 AT_CLEANUP
 
+AT_SETUP([WRITE FROM Japanese literal])
+export LC_ALL=''
+
+AT_DATA([prog.cob], [
+       IDENTIFICATION   DIVISION.
+       PROGRAM-ID.      prog.
+       ENVIRONMENT      DIVISION.
+       INPUT-OUTPUT     SECTION.
+       FILE-CONTROL.
+           SELECT TEST-FILE ASSIGN TO "TESTFILE"
+                  ORGANIZATION IS SEQUENTIAL.
+       DATA DIVISION.
+       FILE SECTION.
+       FD  TEST-FILE.
+       01  TEST-DATA     PIC X(36).
+       WORKING-STORAGE SECTION.
+       PROCEDURE DIVISION.
+           OPEN  OUTPUT TEST-FILE.
+           WRITE TEST-DATA FROM  "縄文弥生古墳飛鳥奈良平安鎌倉室町江戸".
+           CLOSE TEST-FILE.
+
+           OPEN INPUT TEST-FILE.
+           READ TEST-FILE
+               NOT AT END
+                   DISPLAY TEST-DATA
+           END-READ.
+           CLOSE TEST-FILE.
+])
+
+AT_CHECK([cobj prog.cob])
+AT_CHECK([java prog > out1.txt])
+AT_CHECK([echo -n '縄文弥生古墳飛鳥奈良平安鎌倉室町江戸
+' | nkf --ic=UTF-8 --oc=Shift_JIS > out2.txt])
+AT_CHECK([diff out1.txt out2.txt])
+
+AT_CLEANUP
+
 #AT_SETUP([Readable string literals])
 #export LC_ALL=''
 ## Older compilers converts string literals "日本語" in COBOL source code