Switch to PCRE2 for regular expressions

.gitattributes

@@ -7,6 +7,6 @@ vendor/** linguist-vendored
 src/lexer.[ch] linguist-generated=true
 src/parser.[ch] linguist-generated=true
 tests/man.test linguist-generated=true
-tests/manonig.test linguist-generated=true
+tests/manpcre2.test linguist-generated=true
 jq.1.prebuilt linguist-generated=true
 docs/Pipfile.lock linguist-generated=true

.github/workflows/ci.yml

@@ -77,18 +77,18 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v6
         with:
-          submodules: true
+          submodules: recursive
       - name: Install packages
         run: |
           sudo apt-get update
           sudo apt-get install -y automake autoconf libtool crossbuild-essential-${{ matrix.arch }}
       - name: Build
         run: |
-          autoreconf -i
+          autoreconf --verbose -i
           ./configure \
             --host=${{ matrix.CC }} \
             --disable-docs \
-            --with-oniguruma=builtin \
+            --with-pcre2=builtin \
             --enable-static \
             --enable-all-static \
             CFLAGS="-O2 -pthread -fstack-protector-all"
@@ -138,7 +138,7 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v6
         with:
-          submodules: true
+          submodules: recursive
       - name: Install packages
         run: |
           # brew update sometimes fails with "Fetching /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask failed!"
@@ -149,11 +149,11 @@ jobs:
           echo "CC=clang -target ${{ matrix.target }}$(uname -r)" >> "$GITHUB_ENV"
       - name: Build
         run: |
-          autoreconf -i
+          autoreconf --no-recursive -i
           ./configure \
             --host="${{ matrix.target }}$(uname -r)" \
             --disable-docs \
-            --with-oniguruma=builtin \
+            --with-pcre2=builtin \
             --disable-shared \
             --enable-static \
             --enable-all-static \
@@ -215,7 +215,7 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v6
         with:
-          submodules: true
+          submodules: recursive
       - uses: msys2/setup-msys2@v2
         with:
           update: true
@@ -230,10 +230,10 @@ jobs:
             toolchain:p
       - name: Build
         run: |
-          autoreconf -i
+          autoreconf --no-recursive -i
           ./configure \
             --disable-docs \
-            --with-oniguruma=builtin \
+            --with-pcre2=builtin \
             --disable-shared \
             --enable-static \
             --enable-all-static \
@@ -243,7 +243,9 @@ jobs:
           cp ./jq.exe jq-${{ env.SUFFIX }}.exe
       - name: Test
         run: |
-          make check VERBOSE=yes
+          # TODO: pcre2 tests do not work in our msys2 env
+          # so use check-am that does not do recursive check
+          make check-am VERBOSE=yes
           git diff --exit-code
       - name: Upload Test Logs
         if: ${{ failure() }}
@@ -268,7 +270,7 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v6
         with:
-          submodules: true
+          submodules: recursive
       - name: Install packages
         run: |
           sudo apt-get update -qq
@@ -278,7 +280,7 @@ jobs:
           autoreconf -i
           ./configure \
             --disable-docs \
-            --with-oniguruma=builtin
+            --with-pcre2=builtin
           make distcheck
           make dist dist-zip
           git diff --exit-code
@@ -383,7 +385,7 @@ jobs:
         uses: actions/download-artifact@v8
         with:
           pattern: jq-*
-          merge-multiple: true
+          merge-multiple: recursive
       - name: Upload release
         env:
           TAG_NAME: ${{ github.ref_name }}

.github/workflows/decnum.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v6
         with:
-          submodules: true
+          submodules: recursive
       - name: Install packages
         run: |
           sudo apt-get update

.github/workflows/manpage.yml

@@ -1,18 +1,18 @@
-name: Building man page, man.test, manonig.test
+name: Building man page, man.test, manpcre2.test
 on:
   push:
     paths:
       - '.github/workflows/manpage.yml'
       - 'docs/**'
       - 'tests/man.test'
-      - 'tests/manonig.test'
+      - 'tests/manpcre2.test'
       - 'jq.1.prebuilt'
   pull_request:
     paths:
       - '.github/workflows/manpage.yml'
       - 'docs/**'
       - 'tests/man.test'
-      - 'tests/manonig.test'
+      - 'tests/manpcre2.test'
       - 'jq.1.prebuilt'
 
 jobs:
@@ -38,14 +38,14 @@ jobs:
         run: |
           autoreconf -i
           ./configure --enable-docs
-      - name: Build man page, man.test, manonig.test
+      - name: Build man page, man.test, manpcre2.test
         run: |
           mv jq.1.prebuilt jq.1.old
-          rm -f tests/man.test manonig.test
-          make jq.1.prebuilt tests/man.test tests/manonig.test
-      - name: Make sure that jq.1.prebuilt, man.test, manonig.test are up to date
+          rm -f tests/man.test manpcre2.test
+          make jq.1.prebuilt tests/man.test tests/manpcre2.test
+      - name: Make sure that jq.1.prebuilt, man.test, manpcre2.test are up to date
         run: |
-          git diff --exit-code tests/man.test tests/manonig.test
+          git diff --exit-code tests/man.test tests/manpcre2.test
           # skip build date in jq.1.prebuilt
           test -s jq.1.prebuilt
           diff -- <(tail -n +3 jq.1.old) <(tail -n +3 jq.1.prebuilt)

.github/workflows/oniguruma.yml → .github/workflows/pcre2.yml

@@ -1,12 +1,12 @@
-name: oniguruma
+name: pcre2
 on:
   push:
     branches:
       - master
   pull_request:
 
-# Since builtin oniguruma is tested in the CI workflow,
-# we test other options for --with-oniguruma here.
+# Since builtin pcre2 is tested in the CI workflow,
+# we test other options for --with-pcre2 here.
 jobs:
   installed:
     runs-on: ubuntu-24.04
@@ -16,14 +16,14 @@ jobs:
       - name: Install packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y automake autoconf libtool valgrind libonig-dev
+          sudo apt-get install -y automake autoconf libtool valgrind libpcre2-dev libpcre2-8-0
       - name: Build
         run: |
           autoreconf -i
           ./configure \
             --disable-docs \
             --enable-valgrind \
-            --with-oniguruma=yes
+            --with-pcre2=yes
           make -j"$(nproc)"
           file ./jq
       - name: Test
@@ -35,7 +35,7 @@ jobs:
         if: ${{ failure() }}
         uses: actions/upload-artifact@v7
         with:
-          name: test-logs-oniguruma-installed
+          name: test-logs-pcre-installed
           retention-days: 7
           path: |
             test-suite.log
@@ -56,7 +56,7 @@ jobs:
           ./configure \
             --disable-docs \
             --enable-valgrind \
-            --with-oniguruma=no
+            --with-pcre2=no
           make -j"$(nproc)"
           file ./jq
       - name: Test
@@ -68,7 +68,7 @@ jobs:
         if: ${{ failure() }}
         uses: actions/upload-artifact@v7
         with:
-          name: test-logs-oniguruma-disabled
+          name: test-logs-pcre-disabled
           retention-days: 7
           path: |
             test-suite.log

.github/workflows/scanbuild.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v6
         with:
-          submodules: true
+          submodules: recursive
       - name: Install packages
         run: |
           sudo apt-get update -qq

.github/workflows/valgrind.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v6
         with:
-          submodules: true
+          submodules: recursive
       - name: Install packages
         run: |
           sudo apt-get update
@@ -23,7 +23,7 @@ jobs:
           ./configure \
             --disable-docs \
             --enable-valgrind \
-            --with-oniguruma=builtin
+            --with-pcre2=builtin
           make -j"$(nproc)"
           file ./jq
       - name: Test

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "vendor/oniguruma"]
-	path = vendor/oniguruma
-	url = https://github.com/kkos/oniguruma.git
+[submodule "vendor/pcre2"]
+	path = vendor/pcre2
+	url = https://github.com/PCRE2Project/pcre2.git

Dockerfile

@@ -19,7 +19,7 @@ COPY . /app
 RUN autoreconf -i \
  && ./configure \
       --disable-docs \
-      --with-oniguruma=builtin \
+      --with-pcre2=builtin \
       --enable-static \
       --enable-all-static \
       --prefix=/usr/local \

Makefile.am

@@ -56,7 +56,7 @@ AM_YFLAGS = --warnings=all -Wno-yacc -d
 lib_LTLIBRARIES = libjq.la
 libjq_la_SOURCES = ${LIBJQ_SRC}
 libjq_la_LIBADD = -lm
-libjq_la_LDFLAGS = $(onig_LDFLAGS) -export-symbols-regex '^j[qv]_' -version-info 1:4:0
+libjq_la_LDFLAGS = $(pcre2_LDFLAGS) $(pcre2_LDFLAGS) -export-symbols-regex '^j[qv]_' -version-info 1:4:0
 
 if WIN32
 libjq_la_LIBADD += -lshlwapi
@@ -158,7 +158,7 @@ check_DATA = tests/man.test
 # Making changes to the manpage without having the python deps means your
 # tests won't run. If you aren't making changes to the examples, you probably
 # don't care. But if you are, then you need to run the tests anyway.
-tests/man.test tests/manonig.test: $(srcdir)/docs/content/manual/dev/manual.yml
+tests/man.test tests/manopcre2.test: $(srcdir)/docs/content/manual/dev/manual.yml
 if ENABLE_DOCS
 	$(AM_V_GEN) ( cd ${abs_srcdir}/docs && \
 	  $(PIPENV) run python validate_manual_schema.py content/manual/dev/manual.yml && \
@@ -189,18 +189,19 @@ jq.1: jq.1.prebuilt
 
 CLEANFILES += jq.1
 
-### Build oniguruma
+### Build PCRE2
 
-if BUILD_ONIGURUMA
-libjq_la_LIBADD += vendor/oniguruma/src/.libs/libonig.la
-jq_LDADD += vendor/oniguruma/src/.libs/libonig.la
-SUBDIRS = vendor/oniguruma
+if BUILD_PCRE2
+libjq_la_LIBADD += vendor/pcre2/.libs/libpcre2-8.la
+jq_LDADD += vendor/pcre2/.libs/libpcre2-8.la
+SUBDIRS = vendor/pcre2
 endif
 
-AM_CFLAGS += $(onig_CFLAGS)
+AM_CFLAGS += $(pcre2_CFLAGS)
+AM_CFLAGS += $(pcre2_CFLAGS)
 
-if WITH_ONIGURUMA
-TESTS += tests/onigtest tests/manonigtest
+if WITH_PCRE2
+TESTS += tests/pcre2test tests/manpcre2test
 endif
 
 ### Packaging
@@ -227,14 +228,14 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         tests/modules/test_bind_order0.jq                               \
         tests/modules/test_bind_order1.jq                               \
         tests/modules/test_bind_order2.jq                               \
-        tests/onig.supp tests/local.supp                                \
+        tests/pcre2.supp tests/local.supp                                \
         tests/setup tests/torture/input0.json                           \
-        tests/optional.test tests/man.test tests/manonig.test           \
-        tests/jq.test tests/onig.test tests/base64.test tests/uri.test  \
+        tests/optional.test tests/man.test tests/manpcre2.test           \
+        tests/jq.test tests/pcre2.test tests/pcre2.test tests/base64.test tests/uri.test  \
         tests/jq-f-test.sh                                              \
         tests/no-main-program.jq tests/yes-main-program.jq
 
-AM_DISTCHECK_CONFIGURE_FLAGS=--with-oniguruma=builtin
+AM_DISTCHECK_CONFIGURE_FLAGS=--with-pcre2=builtin
 
 # README.md is expected in GitHub projects, good stuff in it, so we'll
 # distribute it and install it with the package in the doc directory.

README.md

@@ -43,10 +43,10 @@ docker run --rm -i -v "$PWD:$PWD" -w "$PWD" ghcr.io/jqlang/jq:latest '.version'
 #### Instructions
 
 ```console
-git submodule update --init    # if building from git to get oniguruma
-autoreconf -i                  # if building from git
-./configure --with-oniguruma=builtin
-make clean                     # if upgrading from a version previously built from source
+git submodule update --init --recursive # if building from git to get pcre2
+autoreconf -i                           # if building from git
+./configure --with-pcre2=builtin        # build with builtin pcre2
+make clean                              # if upgrading from a version previously built from source
 make -j8
 make check
 sudo make install