Implement an encoding switcher of DISPLAY/ACCEPT statements (#590)

Author: yutaro-sakamoto

.github/workflows/test-other.yml

@@ -50,22 +50,17 @@ jobs:
         run: |
           dnf -y update
           dnf install -y gcc make diffutils glibc-gconv-extra unzip wget
-          wget "http://sourceforge.jp/frs/redir.php?m=jaist&f=%2Fnkf%2F59912%2Fnkf-2.1.3.tar.gz" -O nkf-2.1.3.tar.gz --no-check-certificate
+          wget "https://github.com/nurse/nkf/archive/refs/tags/v2_1_3.tar.gz" -O nkf-2.1.3.tar.gz
           tar zxf nkf-2.1.3.tar.gz
-          cd nkf-2.1.3
+          cd nkf-2_1_3
           make
           make install
 
       - name: Install dependencies on Amazon Linux 2023
         if: inputs.os == 'amazonlinux:2023'
         run: |
           dnf -y update
-          dnf install -y gcc make diffutils tar gzip unzip wget
-          wget "http://sourceforge.jp/frs/redir.php?m=jaist&f=%2Fnkf%2F59912%2Fnkf-2.1.3.tar.gz" -O nkf-2.1.3.tar.gz --no-check-certificate
-          tar zxf nkf-2.1.3.tar.gz
-          cd nkf-2.1.3
-          make
-          make install
+          dnf install -y gcc make diffutils tar gzip unzip wget nkf
 
       - name: Install Java
         if: inputs.os == 'amazonlinux:2023'

libcobj/app/src/main/java/jp/osscons/opensourcecobol/libcobj/common/CobolEncoding.java

@@ -0,0 +1,15 @@
+package jp.osscons.opensourcecobol.libcobj.common;
+
+/**
+ * 文字コードを表す列挙型
+ */
+public enum CobolEncoding {
+    /**
+     * UTF-8を表す
+     */
+    UTF8,
+    /**
+     * Shift_JISを表す
+     */
+    SHIFT_JIS,
+};

libcobj/app/src/main/java/jp/osscons/opensourcecobol/libcobj/common/CobolUtil.java

@@ -72,6 +72,9 @@ public class CobolUtil {
     /** TDOD: 準備中 */
     public static int fileSeqWriteBufferSize = 10;
 
+    /** DISPLAY/ACCEPT文によるデータ出力時のエンコーディング */
+    public static CobolEncoding terminalEncoding = CobolEncoding.SHIFT_JIS;
+
     private static boolean lineTrace = false;
 
     /** TDOD: 準備中 */
@@ -349,6 +352,16 @@ public static void cob_init(String[] argv, boolean cobInitialized) {
                 CobolUtil.fileSeqWriteBufferSize = size;
             }
         }
+
+        s = System.getenv("COB_TERMINAL_ENCODING");
+        CobolUtil.terminalEncoding = CobolEncoding.SHIFT_JIS;
+        if (s != null) {
+            Pattern p = Pattern.compile("[uU][tT][fF][_-]?8");
+            Matcher m = p.matcher(s);
+            if (m.matches()) {
+                CobolUtil.terminalEncoding = CobolEncoding.UTF8;
+            }
+        }
     }
 
     // libcob/common.cとcob_localtime

libcobj/app/src/main/java/jp/osscons/opensourcecobol/libcobj/termio/CobolTerminal.java

@@ -18,11 +18,16 @@
  */
 package jp.osscons.opensourcecobol.libcobj.termio;
 
+import java.io.InputStreamReader;
 import java.io.PrintStream;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.UnsupportedCharsetException;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Scanner;
+import jp.osscons.opensourcecobol.libcobj.common.CobolEncoding;
 import jp.osscons.opensourcecobol.libcobj.common.CobolModule;
 import jp.osscons.opensourcecobol.libcobj.common.CobolUtil;
 import jp.osscons.opensourcecobol.libcobj.data.AbstractCobolField;
@@ -68,7 +73,16 @@ public static void display(boolean dispStdout, boolean newline, AbstractCobolFie
 
     private static void displayAlnum(AbstractCobolField f, PrintStream stream) {
         CobolDataStorage storage = f.getDataStorage();
-        stream.write(storage.getRefOfData(), storage.getIndex(), f.getSize());
+        if (CobolUtil.terminalEncoding == CobolEncoding.UTF8) {
+            byte[] utf8Bytes =
+                    new String(
+                                    storage.getByteArrayRef(0, f.getSize()),
+                                    AbstractCobolField.charSetSJIS)
+                            .getBytes(StandardCharsets.UTF_8);
+            stream.write(utf8Bytes, 0, utf8Bytes.length);
+        } else {
+            stream.write(storage.getRefOfData(), storage.getIndex(), f.getSize());
+        }
     }
 
     /**
@@ -108,7 +122,18 @@ public static void display(
     public static void accept(AbstractCobolField f) {
         try {
             if (scan == null) {
-                scan = new Scanner(System.in);
+                if (CobolUtil.terminalEncoding == CobolEncoding.UTF8) {
+                    scan = new Scanner(new InputStreamReader(System.in, StandardCharsets.UTF_8));
+                } else {
+                    try {
+                        scan =
+                                new Scanner(
+                                        new InputStreamReader(
+                                                System.in, Charset.forName("Shift_JIS")));
+                    } catch (UnsupportedCharsetException e) {
+                        scan = new Scanner(System.in);
+                    }
+                }
             }
 
             String input = scan.nextLine();

tests/i18n_sjis.src/data/in-sjis.txt

@@ -0,0 +1,3 @@
+�d�H��
+������
+����Š����H

tests/i18n_sjis.src/data/in-utf8.txt

@@ -0,0 +1,3 @@
+重工業
+給水所
+洗濯板滑走路

tests/i18n_sjis.src/data/out-sjis.txt

@@ -0,0 +1,7 @@
+���{��
+�R����
+�㒆��
+���w�Z
+�d�H��
+������
+�����

tests/i18n_sjis.src/data/out-utf8.txt

@@ -0,0 +1,7 @@
+日本語
+山梨県
+上中下
+小学校
+重工業
+給水所
+洗濯板

tests/i18n_sjis.src/pic-x.at

@@ -541,4 +541,44 @@ AT_CHECK([grep '
 AT_CHECK([grep '�2' < prog3.java > /dev/null])
 AT_CHECK([grep '�3' < prog3.java > /dev/null])
 
-AT_CLEANUP
+AT_CLEANUP
+
+AT_SETUP([DISPLAY/ACCEPT encodings])
+
+AT_DATA([prog.cbl], [
+       IDENTIFICATION DIVISION.
+       PROGRAM-ID. prog.
+       ENVIRONMENT DIVISION.
+       DATA DIVISION.
+       WORKING-STORAGE SECTION.
+       01 X1 PIC X(6) VALUE "�R����".
+       01 N1 PIC N(3) VALUE "�㒆��".
+       01 G1.
+         03 X2 PIC X(6) VALUE "���w�Z".
+         01 N2 PIC N(3) VALUE "���ؗ�".
+       PROCEDURE DIVISION.
+           DISPLAY "���{��".
+           DISPLAY X1.
+           DISPLAY N1.
+           DISPLAY G1.
+
+           ACCEPT X1.
+           DISPLAY X1.
+           ACCEPT N1.
+           DISPLAY N1.
+           ACCEPT G1.
+           DISPLAY G1.
+           STOP RUN.
+])
+
+AT_CHECK([cobj prog.cbl])
+
+AT_CHECK([java prog                                 < ../../i18n_sjis.src/data/in-sjis.txt > out-none.txt])
+AT_CHECK([COB_TERMINAL_ENCODING=SHIFT_JIS java prog < ../../i18n_sjis.src/data/in-sjis.txt > out-sjis.txt])
+AT_CHECK([COB_TERMINAL_ENCODING=UTF-8     java prog < ../../i18n_sjis.src/data/in-utf8.txt > out-utf8.txt])
+
+AT_CHECK([diff out-none.txt ../../i18n_sjis.src/data/out-sjis.txt])
+AT_CHECK([diff out-sjis.txt ../../i18n_sjis.src/data/out-sjis.txt])
+AT_CHECK([diff out-utf8.txt ../../i18n_sjis.src/data/out-utf8.txt])
+
+AT_CLEANUP