Improve package scan performance

Dockerfile

@@ -38,9 +38,11 @@ WORKDIR /scancode-toolkit
 COPY . /scancode-toolkit
 
 # Initial configuration using ./configure, scancode-reindex-licenses to build
-# the base license index
+# the base license index and scancode-reindex-package-patterns to build the 
+# package patterns cache
 RUN ./configure \
- && ./venv/bin/scancode-reindex-licenses
+ && ./venv/bin/scancode-reindex-licenses \
+ && ./venv/bin/scancode-reindex-package-patterns
 
 # Add scancode to path
 ENV PATH=/scancode-toolkit:$PATH

azure-pipelines.yml

@@ -145,14 +145,6 @@ jobs:
           test_suites:
               all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2
 
-    - template: etc/ci/azure-posix.yml
-      parameters:
-          job_name: macos13_cpython
-          image_name: macOS-13
-          python_versions: ['3.10', '3.11', '3.12', '3.13']
-          test_suites:
-              all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2
-
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2025_cpython
@@ -220,14 +212,6 @@ jobs:
           test_suites:
               all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
 
-    - template: etc/ci/azure-posix.yml
-      parameters:
-          job_name: macos13_cpython_latest_from_pip
-          image_name: macos-13
-          python_versions: ['3.10', '3.11', '3.12', '3.13']
-          test_suites:
-              all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
-
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython_latest_from_pip

docs/source/rst-snippets/cli-basic-options.rst

@@ -33,6 +33,12 @@ documenting a program's options. For example:
 --system-package             Scan ``<input>`` for installed system package
                              databases.
 
+--package-in-compiled        Scan compiled executable binaries such as ELF,
+                             WinpE and Mach-O files, looking for structured
+                             package and dependency metadata. Note that looking for
+                             packages in binaries makes package scan slower.
+                             Currently supported compiled binaries: Go, Rust.
+
 --package-only               Faster package scan, scanning ``<input>`` for
                              system and application packages, only for package
                              metadata. This option is skipping

etc/release/scancode-create-pypi-wheel.sh

@@ -19,6 +19,7 @@ set -e
 
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/scancode-reindex-package-patterns
 
 python_tag=$( python -c "import platform;print(f\"cp{''.join(platform.python_version_tuple()[:2])}\")" )
 

etc/release/scancode-create-release-app-linux.sh

@@ -65,6 +65,7 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/scancode-reindex-package-patterns
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

etc/release/scancode-create-release-app-macos.sh

@@ -63,6 +63,7 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/scancode-reindex-package-patterns
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

etc/release/scancode-create-release-app-windows.sh

@@ -62,6 +62,7 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/scancode-reindex-package-patterns
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

requirements.txt

@@ -40,6 +40,7 @@ license-expression==30.4.4
 lxml==5.4.0
 MarkupSafe==3.0.2
 more-itertools==10.7.0
+multiregex==2.0.3
 normality==2.6.1
 packageurl-python==0.17.1
 packaging==25.0

setup-mini.cfg

@@ -89,6 +89,7 @@ install_requires =
     license_expression >= 30.4.4
     lxml >= 5.4.0
     MarkupSafe >= 2.1.2
+    multiregex >= 2.0.3    
     normality <= 2.6.1
     packageurl_python >= 0.9.0
     packvers >= 21.0.0
@@ -156,6 +157,7 @@ packages =
 console_scripts =
     scancode = scancode.cli:scancode
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
+    scancode-reindex-package-patterns = packagedcode.cache:cache_package_patterns
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
     add-required-phrases = licensedcode.required_phrases:add_required_phrases

setup.cfg

@@ -74,6 +74,7 @@ install_requires =
     colorama >= 0.3.9
     commoncode >= 32.4.0
     container-inspector >= 31.0.0
+    cyseq >= 0.0.2
     debian-inspector >= 31.1.0
     dparse2 >= 0.7.0
     fasteners
@@ -90,6 +91,7 @@ install_requires =
     license_expression >= 30.4.4
     lxml >= 5.4.0
     MarkupSafe >= 2.1.2
+    multiregex >= 2.0.3    
     normality <= 2.6.1
     packageurl_python >= 0.9.0
     packvers >= 21.0.0
@@ -116,7 +118,6 @@ install_requires =
     typecode >= 30.0.1
     typecode[full] >= 30.0.1
     extractcode[full] >= 31.0.0
-    cyseq >= 0.0.2
 
 
 [options.packages.find]
@@ -158,6 +159,7 @@ packages =
 console_scripts =
     scancode = scancode.cli:scancode
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
+    scancode-reindex-package-patterns = packagedcode.cache:cache_package_patterns
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
     add-required-phrases = licensedcode.required_phrases:add_required_phrases

src/licensedcode/cache.py

@@ -34,7 +34,6 @@
 LICENSE_INDEX_DIR = 'license_index'
 LICENSE_INDEX_FILENAME = 'index_cache'
 LICENSE_LOCKFILE_NAME = 'scancode_license_index_lockfile'
-LICENSE_CHECKSUM_FILE = 'scancode_license_index_tree_checksums'
 
 
 class LicenseCache:

src/licensedcode/plugin_license.py

@@ -152,6 +152,9 @@ def setup(self, **kwargs):
         This is a cache warmup such that child process inherit from the
         loaded index.
         """
+        if kwargs.get("package_only"):
+            return
+
         from licensedcode.cache import populate_cache
         populate_cache()
 

src/packagedcode/__init__.py

@@ -246,23 +246,32 @@
         win_reg.InstalledProgramFromDockerUtilityvmSoftwareHandler,
     ]
 
+
+# These handlers are special as they use filetype to
+# detect these compiled binaries instead of datafile path patterns
+# as these are optionally installed, we can skip checking
+# for filetype if these are not available
+PACKAGE_IN_COMPILED_DATAFILE_HANDLERS = []
+
 try:
     from go_inspector.binary import get_go_binary_handler
-    APPLICATION_PACKAGE_DATAFILE_HANDLERS.append(get_go_binary_handler())
+    handler = get_go_binary_handler()
+    PACKAGE_IN_COMPILED_DATAFILE_HANDLERS.append(handler)
 except ImportError:
     pass
 
 try:
     from rust_inspector.packages import get_rust_binary_handler
-    APPLICATION_PACKAGE_DATAFILE_HANDLERS.append(get_rust_binary_handler())
+    handler = get_rust_binary_handler()
+    PACKAGE_IN_COMPILED_DATAFILE_HANDLERS.append(handler)
 except ImportError:
     pass
 
 ALL_DATAFILE_HANDLERS = (
     APPLICATION_PACKAGE_DATAFILE_HANDLERS + [
         p for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS
         if p not in APPLICATION_PACKAGE_DATAFILE_HANDLERS
-    ]
+    ] + PACKAGE_IN_COMPILED_DATAFILE_HANDLERS
 )
 
 # registry of all handler classes keyed by datasource_id