Modified to support COBOL source code written in UTF-8. (#582)

* fix: convert SJIS string literals to ordinary Java string literals

* test: add test suites

* test: add new tests cobol_utf8

* ci: run tests of --enable-utf8 tests

* ci: fix workflows for push events

* ci: disable `fail-fast` of some jobs temporarily

* ci: change pull_request.yml to runu utf8 tests

* ci: fix push.yml and pull-request.yml slightly

* test: covert some misc tests to UTF8 versions and add them to cobol_utf8

* test: change utf8 tests

* ci: run m4 when installing the compiler

* fix: dispatch encodings of javac

* wip: fix slightly

* wip: wip

* fix: update libcobj to encode Strings to SJIS

* fix: add -Wno-parentheses

* test: fix tests

* fix: disable size checks of UTF-8 data

* fix: value clause

* test: fix pic-n.at and pic-x

* fix: typeck.c to suppress checks against inspect statement

* [WIP] fix: Remove the character encoding specification for 'new String'

* [WIP] fix: Fix the byte size calculation for Japanese variable names.

* fix: Fix the code related to the byte size of UTF-8.

* fix: Fix the code related to the byte size of UTF-8.

* fix: Fix the code related to the byte size of UTF-8.

* fix: Remove unnecessary codes

* fix: Minor adjustments related to code generation

* fix: Fix the code generation for strings that are SJIS multibyte.

* fix: Fix Fix the bug about MOVE of literals when UTF-8

* fix: Remove unnecessary comments.

* fix: Fix regarding UTF8 multi byte strings

* fix: Fix regarding UTF8 multi byte strings

* fix: Fix regarding PROGRAM-ID written in UTF8 multibyte

* fix: Fix test scripts

* fix: Clean up codes

* Clean up codes

* fix: Clean up codes

* fix: Minor fix

* fix: fieldToString

* ci: install wget in alma and amazon linux

* fix: Fix a bug regarding PROGRAM-ID with multibyte chars

* Fix: Convert cb_zero_utf8 to cb_zen_zero

---------

Co-authored-by: Yutaro Sakamoto <mail@yutaro-sakamoto.com>
Co-authored-by: unknown <y-sakamoto@TSH.LOCAL>
Co-authored-by: Yutaro Sakamoto <yutaro-sakamoto@yutaro-sakamoto.com>
Co-authored-by: Yutaro Sakamoto <y-sakamoto@tsh-world.co.jp>

Author: 4 people

.github/workflows/build.yml

@@ -6,6 +6,10 @@ on:
       os:
         required: true
         type: string
+      configure-args:
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -32,26 +36,27 @@ jobs:
         if: inputs.os == 'ubuntu:24.04'
         run: |
           apt-get update -y
-          apt-get install -y build-essential gettext autoconf
+          apt-get install -y build-essential gettext autoconf bison flex
 
       - name: Install dependencies on AlmaLinux 9
         if: inputs.os == 'almalinux:9'
         run: |
           dnf -y update
-          dnf install -y gcc make autoconf diffutils gettext
+          dnf install -y gcc make autoconf diffutils gettext bison flex
 
       - name: Install dependencies on Amazon Linux 2023
         if: inputs.os == 'amazonlinux:2023'
         run: |
           dnf -y update
-          dnf install -y java-21-amazon-corretto-devel gcc make autoconf diffutils gettext tar gzip
+          dnf install -y java-21-amazon-corretto-devel gcc make autoconf diffutils gettext tar gzip bison flex
 
       - name: Checkout opensource COBOL 4J
         uses: actions/checkout@v4
 
       - name: Install opensource COBOL 4J
         run: |
-          ./configure --prefix=/usr/ CFLAGS=-Werror
+          ./configure --prefix=/usr/ CFLAGS=-Werror ${{ inputs.configure-args }}
+          touch cobj/*.m4
           make
           echo "ARTIFACT_NAME=${{ inputs.os }}" | sed 's/:/-/g' >> "$GITHUB_ENV"
 
@@ -64,5 +69,5 @@ jobs:
       - name: Upload an artifact
         uses: actions/upload-artifact@v4
         with:
-          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}
+          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}-opt_${{ inputs.configure-args }}
           path: opensourcecobol4j.tar.gz

.github/workflows/pull-request.yml

@@ -31,6 +31,16 @@ jobs:
     with:
       os: ${{ matrix.os }}
 
+  build-utf8:
+    needs: check-workflows
+    strategy:
+      matrix:
+        os: ["ubuntu:24.04", "almalinux:9", "amazonlinux:2023"]
+    uses: ./.github/workflows/build.yml
+    with:
+      os: ${{ matrix.os }}
+      configure-args: --enable-utf8
+
   run-test-other:
     needs: build
     strategy:
@@ -50,6 +60,28 @@ jobs:
       test-name: ${{ matrix.test_name }}
       os: ${{ matrix.os }}
 
+  run-test-other-utf8:
+    needs: build-utf8
+    strategy:
+      fail-fast: false
+      matrix:
+        test_name:
+          - "command-line-options"
+          - "data-rep"
+          - "cobol_utf8"
+          #- "i18n_utf8"
+          - "jp-compat"
+          - "run"
+          - "syntax"
+          - "cobj-idx"
+          #- "misc"
+        os: ["ubuntu:24.04", "almalinux:9", "amazonlinux:2023"]
+    uses: ./.github/workflows/test-other.yml
+    with:
+      test-name: ${{ matrix.test_name }}
+      os: ${{ matrix.os }}
+      configure-args: --enable-utf8
+
   run-test-cobj-api:
     needs: build
     strategy:
@@ -71,6 +103,20 @@ jobs:
       check-result: true
       os: ${{ matrix.os }}
 
+  run-test-nist-utf8:
+    needs: build-utf8
+    strategy:
+      fail-fast: false
+      matrix:
+        test_name: ["IC", "IF", "IX", "NC", "OB", "RL", "SG", "SM", "SQ", "ST"]
+        os: ["ubuntu:24.04", "almalinux:9", "amazonlinux:2023"]
+    uses: ./.github/workflows/test-nist.yml
+    with:
+      test-name: ${{ matrix.test_name }}
+      check-result: true
+      os: ${{ matrix.os }}
+      configure-args: --enable-utf8
+
   run-test-nist-extra:
     needs: build
     strategy:

.github/workflows/push.yml

@@ -30,9 +30,20 @@ jobs:
     with:
       os: ${{ matrix.os }}
 
+  build-utf8:
+    needs: check-workflows
+    strategy:
+      matrix:
+        os: ["ubuntu:24.04"]
+    uses: ./.github/workflows/build.yml
+    with:
+      os: ${{ matrix.os }}
+      configure-args: --enable-utf8
+
   run-test-other:
     needs: build
     strategy:
+      fail-fast: false
       matrix:
         test_name:
           - "command-line-options"
@@ -48,6 +59,28 @@ jobs:
     with:
       test-name: ${{ matrix.test_name }}
       os: ${{ matrix.os }}
+  
+  run-test-other-utf8:
+    needs: build-utf8
+    strategy:
+      fail-fast: false
+      matrix:
+        test_name:
+          - "command-line-options"
+          - "data-rep"
+          - "cobol_utf8"
+          #- "i18n_utf8"
+          - "jp-compat"
+          - "run"
+          - "syntax"
+          - "cobj-idx"
+          #- "misc"
+        os: ["ubuntu:24.04"]
+    uses: ./.github/workflows/test-other.yml
+    with:
+      test-name: ${{ matrix.test_name }}
+      os: ${{ matrix.os }}
+      configure-args: --enable-utf8
 
   run-test-cobj-api:
     needs: build
@@ -61,6 +94,20 @@ jobs:
   run-test-nist:
     needs: build
     strategy:
+      fail-fast: false
+      matrix:
+        test_name: ["IC", "IF", "IX", "NC", "OB", "RL", "SG", "SM", "SQ", "ST"]
+        os: ["ubuntu:24.04"]
+    uses: ./.github/workflows/test-nist.yml
+    with:
+      test-name: ${{ matrix.test_name }}
+      check-result: true
+      os: ${{ matrix.os }}
+
+  run-test-nist-utf8:
+    needs: build-utf8
+    strategy:
+      fail-fast: false
       matrix:
         test_name: ["IC", "IF", "IX", "NC", "OB", "RL", "SG", "SM", "SQ", "ST"]
         os: ["ubuntu:24.04"]
@@ -69,6 +116,7 @@ jobs:
       test-name: ${{ matrix.test_name }}
       check-result: true
       os: ${{ matrix.os }}
+      configure-args: --enable-utf8
 
   run-test-nist-extra:
     needs: build

.github/workflows/test-cobj-api.yml

@@ -6,6 +6,10 @@ on:
       os:
         required: true
         type: string
+      configure-args:
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -24,7 +28,7 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}
+          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}-opt_${{ inputs.configure-args }}
 
       - uses: actions/setup-java@v4
         if: inputs.os != 'amazonlinux:2023'

.github/workflows/test-nist.yml

@@ -12,6 +12,10 @@ on:
       os:
         required: true
         type: string
+      configure-args:
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -30,7 +34,7 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}
+          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}-opt_${{ inputs.configure-args }}
 
       - name: Install Java
         uses: actions/setup-java@v4

.github/workflows/test-other.yml

@@ -9,6 +9,10 @@ on:
       os:
         required: true
         type: string
+      configure-args:
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -27,7 +31,7 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}
+          name: opensourcecobol4j-${{ env.ARTIFACT_NAME }}-opt_${{ inputs.configure-args }}
 
       - uses: actions/setup-java@v4
         if: inputs.os != 'amazonlinux:2023'
@@ -39,19 +43,29 @@ jobs:
         if: inputs.os == 'ubuntu:24.04'
         run: |
           apt-get update -y
-          apt-get install -y build-essential unzip
+          apt-get install -y build-essential unzip nkf
 
       - name: Install dependencies on AlmaLinux 9
         if: inputs.os == 'almalinux:9'
         run: |
           dnf -y update
-          dnf install -y gcc make diffutils glibc-gconv-extra unzip
+          dnf install -y gcc make diffutils glibc-gconv-extra unzip wget
+          wget "http://sourceforge.jp/frs/redir.php?m=jaist&f=%2Fnkf%2F59912%2Fnkf-2.1.3.tar.gz" -O nkf-2.1.3.tar.gz --no-check-certificate
+          tar zxf nkf-2.1.3.tar.gz
+          cd nkf-2.1.3
+          make
+          make install
 
       - name: Install dependencies on Amazon Linux 2023
         if: inputs.os == 'amazonlinux:2023'
         run: |
           dnf -y update
-          dnf install -y gcc make diffutils tar gzip unzip
+          dnf install -y gcc make diffutils tar gzip unzip wget
+          wget "http://sourceforge.jp/frs/redir.php?m=jaist&f=%2Fnkf%2F59912%2Fnkf-2.1.3.tar.gz" -O nkf-2.1.3.tar.gz --no-check-certificate
+          tar zxf nkf-2.1.3.tar.gz
+          cd nkf-2.1.3
+          make
+          make install
 
       - name: Install Java
         if: inputs.os == 'amazonlinux:2023'

.gitignore

@@ -49,6 +49,7 @@ tests/command-line-options
 tests/data-rep
 tests/i18n_sjis
 tests/i18n_utf8
+tests/cobol_utf8
 tests/jp-compat
 tests/cobj-idx
 tests/misc

cobj/cobj.c

@@ -329,6 +329,12 @@ static const struct option long_options[] = {
 #undef CB_WARNDEF
     {NULL, 0, NULL, 0}};
 
+#ifdef I18N_UTF8
+static const char *JAVAC_ENCODING = "UTF-8";
+#else
+static const char *JAVAC_ENCODING = "SJIS";
+#endif
+
 static const char *cob_cc;                    /* gcc */
 static char cob_java_flags[COB_SMALL_BUFF];   /* -I... */
 static char cob_libs[COB_MEDIUM_BUFF];        /* -L... -lcob */
@@ -664,6 +670,43 @@ void sjis_spc_to_ascii(char *str) {
 }
 #endif /*I18N_UTF8*/
 
+#ifdef I18N_UTF8
+size_t utf8_calc_sjis_size(const unsigned char *p, int len) {
+  const unsigned char *ub = p + len;
+  int char_size = 0;
+  size_t name_size = 0;
+  while (p < ub) {
+    char_size = COB_U8BYTE_1(*p);
+    if (char_size == 1) {
+      name_size += 1;
+      p++;
+    } else if (char_size == 3 && utf8_hankaku_kana(p)) {
+      name_size += 1;
+      p += char_size;
+    } else {
+      name_size += 2;
+      p += char_size;
+    }
+  }
+  return name_size;
+}
+
+int utf8_hankaku_kana(const unsigned char *p) {
+  if (p[0] == 0xef) {
+    if (p[1] == 0xbd) {
+      if (p[2] >= 0xa1 && p[2] <= 0xbf) {
+        return 1;
+      }
+    } else if (p[1] == 0xbe) {
+      if (p[2] >= 0x80 && p[2] <= 0x9f) {
+        return 1;
+      }
+    }
+  }
+  return 0;
+}
+#endif /*I18N_UTF8*/
+
 /*
  * Local functions
  */
@@ -1772,8 +1815,9 @@ static int process_compile(struct filename *fn) {
 
   char **program_id;
   for (program_id = program_id_list; *program_id; ++program_id) {
-    snprintf(buff, COB_MEDIUM_BUFF, "javac %s -encoding SJIS -d %s %s/%s.java",
-             cob_java_flags, output_name_a, java_source_dir_a, *program_id);
+    snprintf(buff, COB_MEDIUM_BUFF, "javac %s -encoding %s -d %s %s/%s.java",
+             cob_java_flags, JAVAC_ENCODING, output_name_a, java_source_dir_a,
+             *program_id);
     ret = process(buff);
 
     if (ret) {
@@ -2044,8 +2088,8 @@ static int process_build_single_jar() {
 #else
   char remove_cmd[] = "rm -f";
 #endif
-  sprintf(buff, "javac %s -encoding SJIS -d %s %s/*.java", cob_java_flags,
-          output_name_a, java_source_dir_a);
+  sprintf(buff, "javac %s -encoding %s -d %s %s/*.java", cob_java_flags,
+          JAVAC_ENCODING, output_name_a, java_source_dir_a);
 
   ret = process(buff);
   if (ret) {

cobj/cobj.h

@@ -198,6 +198,8 @@ extern size_t utf8_strlen(const unsigned char *p);
 extern int utf8_casecmp(const char *s1, const char *s2);
 extern void utf8_spc_to_ascii(char *);
 extern int utf8_national_length(const unsigned char *str, int len);
+extern size_t utf8_calc_sjis_size(const unsigned char *data, int len);
+extern int utf8_hankaku_kana(const unsigned char *data);
 #else  /*!I18N_UTF8*/
 extern const unsigned char *sjis_pick(const unsigned char *);
 extern size_t sjis_strlen(const unsigned char *);