diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a44cdc1f7..840742005 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -48,11 +48,11 @@ jobs: path: data key: cred-data-${{ hashFiles('checksums.md5') }} - - name: Set up Python 3.10 + - name: Set up Python 3.11 if: steps.cache-data.outputs.cache-hit != 'true' uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - 2026.01.22 with: - python-version: "3.10" + python-version: "3.11" - name: Update PIP run: python -m pip install --upgrade pip @@ -112,10 +112,10 @@ jobs: if: steps.cache-data.outputs.cache-hit == 'true' run: ls -al . && ls -al data - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - 2026.01.22 with: - python-version: "3.10" + python-version: "3.11" - name: Update PIP run: python -m pip install --upgrade pip @@ -177,7 +177,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] + python-version: [ "3.11", "3.12", "3.13", "3.14" ] steps: @@ -407,10 +407,10 @@ jobs: mv data ${{ github.workspace }}/CredData/ mv meta ${{ github.workspace }}/CredData/ - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - 2026.01.22 with: - python-version: "3.10" + python-version: "3.11" - name: Update PIP run: python -m pip install --upgrade pip diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 1de07f4ee..d73a87e78 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -100,10 +100,6 @@ jobs: # # # pylint - - name: Analysing the code with pylint and minimum Python version 3.10 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: pylint --py-version=3.10 --verbose credsweeper - - name: Analysing the code with pylint and minimum Python version 3.11 if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} run: pylint --py-version=3.11 --verbose credsweeper @@ -122,11 +118,6 @@ jobs: # # # mypy - - name: Analysing the code with mypy and minimum Python version 3.10 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - mypy --config-file .mypy.ini --python-version=3.10 credsweeper - - name: Analysing the code with mypy and minimum Python version 3.11 if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} run: | diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 2333c536d..857522d51 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -37,7 +37,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - 2026.01.22 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index ff5fbe47c..9a48ab134 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -31,7 +31,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - 2026.01.22 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebcfcc0a7..c70c12995 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-latest, windows-latest, macos-latest ] - python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] + python-version: [ "3.11", "3.12", "3.13", "3.14" ] steps: @@ -80,16 +80,7 @@ jobs: run: | pip install pytest pytest-random-order deepdiff hypothesis - - name: UnitTest with pytest for Python & ubuntu-latest - if: ${{ matrix.os == 'ubuntu-latest' && ( matrix.python-version == '3.11' || matrix.python-version == '3.12' || matrix.python-version == '3.13' || matrix.python-version == '3.14' ) }} - run: | - # skip app test with stderr checking - [W:onnxruntime:Default, device_discovery.cc:131 GetPciBusId] - pytest --verbose --random-order --random-order-bucket=global \ - -k 'not (test_log_p or test_log_n or test_rules_ml_p or test_rules_ml_n or test_it_works_n or test_no_filters_p or test_external_ml_p or test_external_ml_n or test_import_config_p or test_import_config_n)' \ - tests - - name: UnitTest with pytest - if: ${{ matrix.os != 'ubuntu-latest' || matrix.python-version == '3.10' }} run: | # put the command into one line to use in various OS to avoid processing differences in new line char sequence pytest --verbose --random-order --random-order-bucket=global tests @@ -102,7 +93,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] + python-version: [ "3.11", "3.12", "3.13", "3.14" ] steps: @@ -201,7 +192,7 @@ jobs: path: coverage_html - name: Upload coverage reports to Codecov - if: ${{ matrix.python-version == '3.10' }} + if: ${{ matrix.python-version == '3.11' }} uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 - 2025.12.09 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.readthedocs.yml b/.readthedocs.yml index f15bb114a..8a12ae598 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ version: 2 build: os: "ubuntu-lts-latest" tools: - python: "3.10" + python: "3.11" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/README.md b/README.md index 0df092ba4..1906e5f86 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Full documentation can be found here: ### Main Requirements -- Python 3.10, 3.11, 3.12, 3.13, 3.14 +- Python 3.11, 3.12, 3.13, 3.14 ### Installation diff --git a/action.yml b/action.yml index af021b14b..821c9d034 100644 --- a/action.yml +++ b/action.yml @@ -6,8 +6,8 @@ branding: color: "gray-dark" inputs: python_version: - description: "Python Version. 3.10 - default" - default: "3.10" + description: "Python Version. 3.11 - default" + default: "3.11" required: false path: description: "Path to scan" diff --git a/credsweeper/deep_scanner/mxfile_scanner.py b/credsweeper/deep_scanner/mxfile_scanner.py index 2f2d2988a..62574f22f 100644 --- a/credsweeper/deep_scanner/mxfile_scanner.py +++ b/credsweeper/deep_scanner/mxfile_scanner.py @@ -37,7 +37,7 @@ def data_scan( tree = etree.fromstring(data_provider.text) for element in tree.iter(): if "mxCell" == getattr(element, "tag"): - line_number = element.sourceline + line_number = -1 if element.sourceline is None else int(element.sourceline) attr = getattr(element, "attrib") if attr is None or not (value := attr.get("value")): continue diff --git a/credsweeper/deep_scanner/sqlite3_scanner.py b/credsweeper/deep_scanner/sqlite3_scanner.py index fec86ec82..a6a1772a9 100644 --- a/credsweeper/deep_scanner/sqlite3_scanner.py +++ b/credsweeper/deep_scanner/sqlite3_scanner.py @@ -1,8 +1,5 @@ import logging -import os.path import sqlite3 -import sys -import tempfile from abc import ABC from typing import List, Optional, Tuple, Any, Generator, Union @@ -41,28 +38,9 @@ def __walk(sqlite3db) -> Generator[Tuple[str, Any], None, None]: @staticmethod def walk_sqlite(data: bytes) -> Generator[Tuple[str, Any], None, None]: """Yields data from sqlite3 database""" - if 10 < sys.version_info.minor: - # Added in version 3.11 - with sqlite3.connect(":memory:") as sqlite3db: - sqlite3db.deserialize(data) # type: ignore - yield from Sqlite3Scanner.__walk(sqlite3db) - elif "nt" != os.name: - # a tmpfile has to be used. TODO: remove when 3.10 will deprecate - with tempfile.NamedTemporaryFile(suffix=".sqlite") as t: - t.write(data) - t.flush() - with sqlite3.connect(t.name) as sqlite3db: - yield from Sqlite3Scanner.__walk(sqlite3db) - elif "nt" == os.name: - # windows trick. TODO: remove when 3.10 will deprecate - with tempfile.NamedTemporaryFile(delete=False, suffix=".sqlite") as t: - t.write(data) - t.flush() - sqlite3db = sqlite3.connect(t.name) + with sqlite3.connect(":memory:") as sqlite3db: + sqlite3db.deserialize(data) # type: ignore yield from Sqlite3Scanner.__walk(sqlite3db) - sqlite3db.close() - if os.path.exists(t.name): - os.remove(t.name) def data_scan( self, # diff --git a/docs/requirements.txt b/docs/requirements.txt index db4084547..082aded1a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ -myst_parser[linkify]==4.0.1 -sphinx==8.1.3 -sphinx_rtd_theme==3.0.2 +# Python 3.11.15 +# pip 26.0.1 + +myst_parser[linkify]==5.0.0 +sphinx==9.0.4 +sphinx_rtd_theme==3.1.0 diff --git a/docs/source/credsweeper.common.rst b/docs/source/credsweeper.common.rst index 99a9cdd4d..0682838b9 100644 --- a/docs/source/credsweeper.common.rst +++ b/docs/source/credsweeper.common.rst @@ -9,29 +9,29 @@ credsweeper.common.constants module .. automodule:: credsweeper.common.constants :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.common.keyword\_checklist module -------------------------------------------- .. automodule:: credsweeper.common.keyword_checklist :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.common.keyword\_pattern module ------------------------------------------ .. automodule:: credsweeper.common.keyword_pattern :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.common :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.config.rst b/docs/source/credsweeper.config.rst index a19bbe263..95f2cea9a 100644 --- a/docs/source/credsweeper.config.rst +++ b/docs/source/credsweeper.config.rst @@ -9,13 +9,13 @@ credsweeper.config.config module .. automodule:: credsweeper.config.config :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.config :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.credentials.rst b/docs/source/credsweeper.credentials.rst index 94c1e67c5..c32b2ee1e 100644 --- a/docs/source/credsweeper.credentials.rst +++ b/docs/source/credsweeper.credentials.rst @@ -9,53 +9,53 @@ credsweeper.credentials.augment\_candidates module .. automodule:: credsweeper.credentials.augment_candidates :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.credentials.candidate module ---------------------------------------- .. automodule:: credsweeper.credentials.candidate :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.credentials.candidate\_group\_generator module ---------------------------------------------------------- .. automodule:: credsweeper.credentials.candidate_group_generator :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.credentials.candidate\_key module --------------------------------------------- .. automodule:: credsweeper.credentials.candidate_key :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.credentials.credential\_manager module -------------------------------------------------- .. automodule:: credsweeper.credentials.credential_manager :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.credentials.line\_data module ----------------------------------------- .. automodule:: credsweeper.credentials.line_data :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.credentials :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.deep_scanner.rst b/docs/source/credsweeper.deep_scanner.rst index 8e669c472..cf34ba693 100644 --- a/docs/source/credsweeper.deep_scanner.rst +++ b/docs/source/credsweeper.deep_scanner.rst @@ -9,253 +9,253 @@ credsweeper.deep\_scanner.abstract\_scanner module .. automodule:: credsweeper.deep_scanner.abstract_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.byte\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.byte_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.bzip2\_scanner module ----------------------------------------------- .. automodule:: credsweeper.deep_scanner.bzip2_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.csv\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.csv_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.deb\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.deb_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.deep\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.deep_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.docx\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.docx_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.eml\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.eml_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.encoder\_scanner module ------------------------------------------------- .. automodule:: credsweeper.deep_scanner.encoder_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.gzip\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.gzip_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.html\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.html_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.jclass\_scanner module ------------------------------------------------ .. automodule:: credsweeper.deep_scanner.jclass_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.jks\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.jks_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.lang\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.lang_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.lzma\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.lzma_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.mxfile\_scanner module ------------------------------------------------ .. automodule:: credsweeper.deep_scanner.mxfile_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.patch\_scanner module ----------------------------------------------- .. automodule:: credsweeper.deep_scanner.patch_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.pdf\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.pdf_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.pkcs\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.pkcs_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.png\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.png_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.pptx\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.pptx_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.rpm\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.rpm_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.rtf\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.rtf_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.sqlite3\_scanner module ------------------------------------------------- .. automodule:: credsweeper.deep_scanner.sqlite3_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.strings\_scanner module ------------------------------------------------- .. automodule:: credsweeper.deep_scanner.strings_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.tar\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.tar_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.tmx\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.tmx_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.xlsx\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.xlsx_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.xml\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.xml_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.zip\_scanner module --------------------------------------------- .. automodule:: credsweeper.deep_scanner.zip_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.deep\_scanner.zlib\_scanner module ---------------------------------------------- .. automodule:: credsweeper.deep_scanner.zlib_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.deep_scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.file_handler.rst b/docs/source/credsweeper.file_handler.rst index d746dba08..9a3f89c9c 100644 --- a/docs/source/credsweeper.file_handler.rst +++ b/docs/source/credsweeper.file_handler.rst @@ -9,109 +9,109 @@ credsweeper.file\_handler.abstract\_provider module .. automodule:: credsweeper.file_handler.abstract_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.analysis\_target module ------------------------------------------------- .. automodule:: credsweeper.file_handler.analysis_target :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.byte\_content\_provider module -------------------------------------------------------- .. automodule:: credsweeper.file_handler.byte_content_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.content\_provider module -------------------------------------------------- .. automodule:: credsweeper.file_handler.content_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.data\_content\_provider module -------------------------------------------------------- .. automodule:: credsweeper.file_handler.data_content_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.descriptor module ------------------------------------------- .. automodule:: credsweeper.file_handler.descriptor :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.diff\_content\_provider module -------------------------------------------------------- .. automodule:: credsweeper.file_handler.diff_content_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.file\_path\_extractor module ------------------------------------------------------ .. automodule:: credsweeper.file_handler.file_path_extractor :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.files\_provider module ------------------------------------------------ .. automodule:: credsweeper.file_handler.files_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.patches\_provider module -------------------------------------------------- .. automodule:: credsweeper.file_handler.patches_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.string\_content\_provider module ---------------------------------------------------------- .. automodule:: credsweeper.file_handler.string_content_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.struct\_content\_provider module ---------------------------------------------------------- .. automodule:: credsweeper.file_handler.struct_content_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.file\_handler.text\_content\_provider module -------------------------------------------------------- .. automodule:: credsweeper.file_handler.text_content_provider :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.file_handler :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.filters.group.rst b/docs/source/credsweeper.filters.group.rst index d14e22e15..78640af36 100644 --- a/docs/source/credsweeper.filters.group.rst +++ b/docs/source/credsweeper.filters.group.rst @@ -9,69 +9,69 @@ credsweeper.filters.group.general\_keyword module .. automodule:: credsweeper.filters.group.general_keyword :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.group.general\_pattern module ------------------------------------------------- .. automodule:: credsweeper.filters.group.general_pattern :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.group.group module -------------------------------------- .. automodule:: credsweeper.filters.group.group :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.group.password\_keyword module -------------------------------------------------- .. automodule:: credsweeper.filters.group.password_keyword :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.group.token\_pattern module ----------------------------------------------- .. automodule:: credsweeper.filters.group.token_pattern :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.group.url\_credentials\_group module -------------------------------------------------------- .. automodule:: credsweeper.filters.group.url_credentials_group :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.group.weird\_base36\_token module ----------------------------------------------------- .. automodule:: credsweeper.filters.group.weird_base36_token :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.group.weird\_base64\_token module ----------------------------------------------------- .. automodule:: credsweeper.filters.group.weird_base64_token :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.filters.group :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.filters.rst b/docs/source/credsweeper.filters.rst index 3d2bbd7a3..8036d337d 100644 --- a/docs/source/credsweeper.filters.rst +++ b/docs/source/credsweeper.filters.rst @@ -17,304 +17,304 @@ credsweeper.filters.filter module .. automodule:: credsweeper.filters.filter :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.line\_git\_binary\_check module --------------------------------------------------- .. automodule:: credsweeper.filters.line_git_binary_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.line\_specific\_key\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.line_specific_key_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.line\_uue\_part\_check module ------------------------------------------------- .. automodule:: credsweeper.filters.line_uue_part_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_allowlist\_check module -------------------------------------------------- .. automodule:: credsweeper.filters.value_allowlist_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_array\_dictionary\_check module ---------------------------------------------------------- .. automodule:: credsweeper.filters.value_array_dictionary_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_atlassian\_token\_check module --------------------------------------------------------- .. automodule:: credsweeper.filters.value_atlassian_token_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_azure\_token\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.value_azure_token_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_base32\_data\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.value_base32_data_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_base64\_data\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.value_base64_data_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_base64\_encoded\_pem\_check module ------------------------------------------------------------- .. automodule:: credsweeper.filters.value_base64_encoded_pem_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_base64\_key\_check module ---------------------------------------------------- .. automodule:: credsweeper.filters.value_base64_key_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_base64\_part\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.value_base64_part_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_basic\_auth\_check module ---------------------------------------------------- .. automodule:: credsweeper.filters.value_basic_auth_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_blocklist\_check module -------------------------------------------------- .. automodule:: credsweeper.filters.value_blocklist_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_camel\_case\_check module ---------------------------------------------------- .. automodule:: credsweeper.filters.value_camel_case_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_dictionary\_keyword\_check module ------------------------------------------------------------ .. automodule:: credsweeper.filters.value_dictionary_keyword_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_discord\_bot\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.value_discord_bot_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_entropy\_base32\_check module -------------------------------------------------------- .. automodule:: credsweeper.filters.value_entropy_base32_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_entropy\_base36\_check module -------------------------------------------------------- .. automodule:: credsweeper.filters.value_entropy_base36_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_entropy\_base64\_check module -------------------------------------------------------- .. automodule:: credsweeper.filters.value_entropy_base64_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_entropy\_base\_check module ------------------------------------------------------ .. automodule:: credsweeper.filters.value_entropy_base_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_file\_path\_check module --------------------------------------------------- .. automodule:: credsweeper.filters.value_file_path_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_github\_check module ----------------------------------------------- .. automodule:: credsweeper.filters.value_github_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_grafana\_check module ------------------------------------------------ .. automodule:: credsweeper.filters.value_grafana_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_grafana\_service\_check module --------------------------------------------------------- .. automodule:: credsweeper.filters.value_grafana_service_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_hex\_number\_check module ---------------------------------------------------- .. automodule:: credsweeper.filters.value_hex_number_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_jfrog\_token\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.value_jfrog_token_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_json\_web\_key\_check module ------------------------------------------------------- .. automodule:: credsweeper.filters.value_json_web_key_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_json\_web\_token\_check module --------------------------------------------------------- .. automodule:: credsweeper.filters.value_json_web_token_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_last\_word\_check module --------------------------------------------------- .. automodule:: credsweeper.filters.value_last_word_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_length\_check module ----------------------------------------------- .. automodule:: credsweeper.filters.value_length_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_method\_check module ----------------------------------------------- .. automodule:: credsweeper.filters.value_method_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_morphemes\_check module -------------------------------------------------- .. automodule:: credsweeper.filters.value_morphemes_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_not\_allowed\_pattern\_check module -------------------------------------------------------------- .. automodule:: credsweeper.filters.value_not_allowed_pattern_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_not\_part\_encoded\_check module ----------------------------------------------------------- .. automodule:: credsweeper.filters.value_not_part_encoded_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_number\_check module ----------------------------------------------- .. automodule:: credsweeper.filters.value_number_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_pattern\_check module ------------------------------------------------ .. automodule:: credsweeper.filters.value_pattern_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_sealed\_secret\_check module ------------------------------------------------------- @@ -329,77 +329,77 @@ credsweeper.filters.value\_search\_check module .. automodule:: credsweeper.filters.value_search_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_similarity\_check module --------------------------------------------------- .. automodule:: credsweeper.filters.value_similarity_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_split\_keyword\_check module ------------------------------------------------------- .. automodule:: credsweeper.filters.value_split_keyword_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_string\_type\_check module ----------------------------------------------------- .. automodule:: credsweeper.filters.value_string_type_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_token\_base32\_check module ------------------------------------------------------ .. automodule:: credsweeper.filters.value_token_base32_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_token\_base36\_check module ------------------------------------------------------ .. automodule:: credsweeper.filters.value_token_base36_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_token\_base64\_check module ------------------------------------------------------ .. automodule:: credsweeper.filters.value_token_base64_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_token\_base\_check module ---------------------------------------------------- .. automodule:: credsweeper.filters.value_token_base_check :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.filters.value\_token\_check module ---------------------------------------------- .. automodule:: credsweeper.filters.value_token_check :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.filters :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.logger.rst b/docs/source/credsweeper.logger.rst index ac8d78535..80f38e16c 100644 --- a/docs/source/credsweeper.logger.rst +++ b/docs/source/credsweeper.logger.rst @@ -9,13 +9,13 @@ credsweeper.logger.logger module .. automodule:: credsweeper.logger.logger :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.logger :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.ml_model.features.rst b/docs/source/credsweeper.ml_model.features.rst index 19b626f67..11faeef37 100644 --- a/docs/source/credsweeper.ml_model.features.rst +++ b/docs/source/credsweeper.ml_model.features.rst @@ -9,141 +9,141 @@ credsweeper.ml\_model.features.entropy\_evaluation module .. automodule:: credsweeper.ml_model.features.entropy_evaluation :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.feature module --------------------------------------------- .. automodule:: credsweeper.ml_model.features.feature :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.file\_extension module ----------------------------------------------------- .. automodule:: credsweeper.ml_model.features.file_extension :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.has\_html\_tag module ---------------------------------------------------- .. automodule:: credsweeper.ml_model.features.has_html_tag :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.is\_secret\_numeric module --------------------------------------------------------- .. automodule:: credsweeper.ml_model.features.is_secret_numeric :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.length\_of\_attribute module ----------------------------------------------------------- .. automodule:: credsweeper.ml_model.features.length_of_attribute :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.morpheme\_dense module ----------------------------------------------------- .. automodule:: credsweeper.ml_model.features.morpheme_dense :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.rule\_name module ------------------------------------------------ .. automodule:: credsweeper.ml_model.features.rule_name :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.rule\_severity module ---------------------------------------------------- .. automodule:: credsweeper.ml_model.features.rule_severity :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.search\_in\_attribute module ----------------------------------------------------------- .. automodule:: credsweeper.ml_model.features.search_in_attribute :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.word\_in module ---------------------------------------------- .. automodule:: credsweeper.ml_model.features.word_in :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.word\_in\_path module ---------------------------------------------------- .. automodule:: credsweeper.ml_model.features.word_in_path :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.word\_in\_postamble module --------------------------------------------------------- .. automodule:: credsweeper.ml_model.features.word_in_postamble :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.word\_in\_preamble module -------------------------------------------------------- .. automodule:: credsweeper.ml_model.features.word_in_preamble :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.word\_in\_transition module ---------------------------------------------------------- .. automodule:: credsweeper.ml_model.features.word_in_transition :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.word\_in\_value module ----------------------------------------------------- .. automodule:: credsweeper.ml_model.features.word_in_value :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.ml\_model.features.word\_in\_variable module -------------------------------------------------------- .. automodule:: credsweeper.ml_model.features.word_in_variable :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.ml_model.features :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.ml_model.rst b/docs/source/credsweeper.ml_model.rst index bbcf3e607..afc192d1c 100644 --- a/docs/source/credsweeper.ml_model.rst +++ b/docs/source/credsweeper.ml_model.rst @@ -17,13 +17,13 @@ credsweeper.ml\_model.ml\_validator module .. automodule:: credsweeper.ml_model.ml_validator :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.ml_model :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.rst b/docs/source/credsweeper.rst index d5a29967e..f657b29ec 100644 --- a/docs/source/credsweeper.rst +++ b/docs/source/credsweeper.rst @@ -1,14 +1,47 @@ -Credsweeper package +credsweeper package =================== -CredSweeper +Subpackages ----------- .. toctree:: :maxdepth: 4 + credsweeper.common + credsweeper.config + credsweeper.credentials + credsweeper.deep_scanner + credsweeper.file_handler + credsweeper.filters + credsweeper.logger + credsweeper.ml_model + credsweeper.rules + credsweeper.scanner + credsweeper.utils + +Submodules +---------- + +credsweeper.app module +---------------------- + .. automodule:: credsweeper.app :members: + :show-inheritance: :undoc-members: + +credsweeper.main module +----------------------- + +.. automodule:: credsweeper.main + :members: :show-inheritance: + :undoc-members: + +Module contents +--------------- +.. automodule:: credsweeper + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.rules.rst b/docs/source/credsweeper.rules.rst index 411c128a7..4adf29aef 100644 --- a/docs/source/credsweeper.rules.rst +++ b/docs/source/credsweeper.rules.rst @@ -9,13 +9,13 @@ credsweeper.rules.rule module .. automodule:: credsweeper.rules.rule :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.rules :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.scanner.rst b/docs/source/credsweeper.scanner.rst index c701fb2c1..71b6776c6 100644 --- a/docs/source/credsweeper.scanner.rst +++ b/docs/source/credsweeper.scanner.rst @@ -17,13 +17,13 @@ credsweeper.scanner.scanner module .. automodule:: credsweeper.scanner.scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.scanner :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.scanner.scan_type.rst b/docs/source/credsweeper.scanner.scan_type.rst index f99aa3e60..6d84f20eb 100644 --- a/docs/source/credsweeper.scanner.scan_type.rst +++ b/docs/source/credsweeper.scanner.scan_type.rst @@ -9,37 +9,37 @@ credsweeper.scanner.scan\_type.multi\_pattern module .. automodule:: credsweeper.scanner.scan_type.multi_pattern :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.scanner.scan\_type.pem\_key\_pattern module ------------------------------------------------------- .. automodule:: credsweeper.scanner.scan_type.pem_key_pattern :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.scanner.scan\_type.scan\_type module ------------------------------------------------ .. automodule:: credsweeper.scanner.scan_type.scan_type :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.scanner.scan\_type.single\_pattern module ----------------------------------------------------- .. automodule:: credsweeper.scanner.scan_type.single_pattern :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.scanner.scan_type :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/credsweeper.utils.rst b/docs/source/credsweeper.utils.rst index f7d42a591..66a30f58b 100644 --- a/docs/source/credsweeper.utils.rst +++ b/docs/source/credsweeper.utils.rst @@ -9,29 +9,29 @@ credsweeper.utils.hop\_stat module .. automodule:: credsweeper.utils.hop_stat :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.utils.pem\_key\_detector module ------------------------------------------- .. automodule:: credsweeper.utils.pem_key_detector :members: - :undoc-members: :show-inheritance: + :undoc-members: credsweeper.utils.util module ----------------------------- .. automodule:: credsweeper.utils.util :members: - :undoc-members: :show-inheritance: + :undoc-members: Module contents --------------- .. automodule:: credsweeper.utils :members: - :undoc-members: :show-inheritance: + :undoc-members: diff --git a/docs/source/guide.rst b/docs/source/guide.rst index 18f21cb91..dd1e46d56 100644 --- a/docs/source/guide.rst +++ b/docs/source/guide.rst @@ -39,73 +39,51 @@ Get all argument list: --diff_path PATH [PATH ...] git diff file to scan --export_config [PATH] - exporting default config to file (default: - config.json) + exporting default config to file (default: config.json) --export_log_config [PATH] - exporting default logger config to file (default: - log.yaml) + exporting default logger config to file (default: log.yaml) --git PATH git repo to scan - --ref REF scan git repo from the ref, otherwise - all branches - were scanned (slow) - --rules PATH path of rule config file (default: - credsweeper/rules/config.yaml). severity:['critical', - 'high', 'medium', 'low', 'info'] type:['keyword', - 'pattern', 'pem_key', 'multi'] - --severity SEVERITY set minimum level for rules to apply ['critical', - 'high', 'medium', 'low', 'info'](default: - 'Severity.INFO', case insensitive) + --ref REF scan git repo from the ref, otherwise - all branches were scanned (slow) + --rules PATH path of rule config file (default: credsweeper/rules/config.yaml). severity:['critical', 'high', 'medium', 'low', 'info'] type:['keyword', 'pattern', 'pem_key', 'multi'] + --severity SEVERITY set minimum level for rules to apply ['critical', 'high', 'medium', 'low', 'info'](default: 'Severity.INFO', case insensitive) --config PATH use custom config (default: built-in) --log_config PATH use custom log config (default: built-in) - --denylist PATH path to a plain text file with lines or secrets to - ignore + --denylist PATH path to a plain text file with lines or secrets to ignore --find-by-ext find files by predefined extension --pedantic, --no-pedantic - process files without extension (default: False) + process files without extension --depth POSITIVE_INT additional recursive search in data (experimental) --no-filters disable filters --doc document-specific scanning --ml_threshold THRESHOLD_OR_FLOAT_OR_ZERO - setup threshold for the ml model. The lower the - threshold - the more credentials will be reported. - Allowed values: float between 0 and 1, or any of - ['lowest', 'low', 'medium', 'high', 'highest'] - (default: medium) + setup threshold for the ml model. The lower the threshold - the more credentials will be reported. Allowed values: float between 0 and 1, or any of ['lowest', 'low', 'medium', 'high', 'highest'] (default: + medium) --ml_batch_size POSITIVE_INT, -b POSITIVE_INT batch size for model inference (default: 16) --ml_config PATH use external config for ml model --ml_model PATH use external ml model - --ml_providers STR comma separated list of providers for onnx - (CPUExecutionProvider is used by default) + --ml_providers STR comma separated list of providers for onnx (CPUExecutionProvider is used by default) --jobs POSITIVE_INT, -j POSITIVE_INT number of parallel processes to use (default: 1) --thrifty, --no-thrifty clear objects after scan to reduce memory consumption - (default: True) - --skip_ignored parse .gitignore files and skip credentials from - ignored objects - --error, --no-error produce error code if credentials are found (default: - False) + --skip_ignored parse .gitignore files and skip credentials from ignored objects + --error, --no-error produce error code if credentials are found --save-json [PATH] save result to json file (default: output.json) --save-xlsx [PATH] save result to xlsx file (default: output.xlsx) --stdout, --no-stdout - print results to stdout (default: True) - --color, --no-color print results with colorization (default: False) + print results to stdout + --color, --no-color print results with colorization --hashed, --no-hashed line, variable, value will be hashed in output - (default: False) --subtext, --no-subtext - line text will be stripped in 128 symbols but value - and variable are kept (default: False) - --sort, --no-sort enable output sorting (default: False) + line text will be stripped in 128 symbols but value and variable are kept + --sort, --no-sort enable output sorting --log LOG_LEVEL, -l LOG_LEVEL - provide logging level of ['DEBUG', 'INFO', 'WARN', - 'WARNING', 'ERROR', 'FATAL', 'CRITICAL', 'SILENCE'] - (default: 'warning', case insensitive) + provide logging level of ['DEBUG', 'INFO', 'WARN', 'WARNING', 'ERROR', 'FATAL', 'CRITICAL', 'SILENCE'] (default: 'warning', case insensitive) --size_limit SIZE_LIMIT - set size limit of files that for scanning (eg. 1GB / - 10MiB / 1000) - --banner show version and crc32 sum of CredSweeper files at - start + set size limit of files that for scanning (eg. 1GB / 10MiB / 1000) + --banner show version and crc32 sum of CredSweeper files at start --version, -V show program's version number and exit .. note:: diff --git a/docs/source/index.rst b/docs/source/index.rst index 1fd98410b..3a2782342 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,74 +1,24 @@ -.. |CredSweeper_logo| image:: ../images/Logo.png - :width: 100 - :alt: Alternative text +.. credsweeper documentation master file, created by + sphinx-quickstart on Sat Apr 4 12:40:30 2026. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. -|CredSweeper_logo| +credsweeper documentation +========================= -Welcome to CredSweeper's documentation! -======================================= +Add your content using ``reStructuredText`` syntax. See the +`reStructuredText `_ +documentation for details. -CredSweeper is a tool to detect credentials in any directories or files. CredSweeper could help users to detect -unwanted exposure of credentials (such as personal information, token, passwords, api keys and etc) in advance. -By scanning lines, filtering, and using AI model as option, CredSweeper reports lines with possible credentials, -where the line is, and expected type of the credential as a result. - -How To Use -========== .. toctree:: - :maxdepth: 2 - - guide - -Installation -============ - -.. toctree:: - :maxdepth: 2 - - install - -Develop -======= - -.. toctree:: - :maxdepth: 2 + :maxdepth: 4 + :caption: Contents: + credsweeper + api develop - - -How to Contribute -================= - -.. toctree:: - :maxdepth: 2 - + guide how_to_contribute - - -Overall architecture -==================== - -.. toctree:: - :maxdepth: 2 - + install overall_architecture - - -API Reference -============= - -If you are looking for information on a specific function, class or method, this part of the documentation is for you. - -.. toctree:: - :maxdepth: 2 - - api - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/install.rst b/docs/source/install.rst index 58a6635fe..1987175fd 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -3,7 +3,7 @@ Installation Currently `CredSweeper` requires the following prerequisites: -* Python version 3.10, 3.11, 3.12 +* Python version 3.11, 3.12, 3.13, 3.14 .. note:: We recommend to use credsweeper in a separate virtual enviroment. Some heave dependencies as Tensorflow diff --git a/experiment/README.md b/experiment/README.md index ebae227ae..65569bbfa 100644 --- a/experiment/README.md +++ b/experiment/README.md @@ -4,7 +4,7 @@ This code will allow you to retrain model on the CredData dataset ## Preparation -- Make sure that you are using Python 3.10 or higher +- Make sure that you are using Python 3.11 or higher - Download CredData dataset diff --git a/experiment/main.py b/experiment/main.py index 05e304b7d..b55a4c15e 100644 --- a/experiment/main.py +++ b/experiment/main.py @@ -3,6 +3,8 @@ import sys from argparse import ArgumentParser, BooleanOptionalAction +os.environ["KERAS_BACKEND"] = "tensorflow" + from train import train diff --git a/experiment/model_config_preprocess.py b/experiment/model_config_preprocess.py index 8575525c1..ca7291f18 100644 --- a/experiment/model_config_preprocess.py +++ b/experiment/model_config_preprocess.py @@ -1,9 +1,11 @@ +import json import mimetypes from typing import Dict import pandas as pd from credsweeper.app import APP_PATH +from credsweeper.common.constants import UTF_8 from credsweeper.utils.util import Util ML_CONFIG_PATH = APP_PATH / "ml_model" / "ml_config.json" @@ -46,7 +48,8 @@ def model_config_preprocess(df_all: pd.DataFrame, doc_target: bool) -> Dict[str, # collect all unknown extensions for error log print(f"UNKNOWN EXTENSION: {extension}", flush=True) unknown_extensions.append(extension) - Util.json_dump(model_config, ML_CONFIG_PATH) + with open(ML_CONFIG_PATH, 'w', encoding=UTF_8) as f: + json.dump(model_config, f, indent=4, sort_keys=True, ensure_ascii=True) if known_extensions != set(x["kwargs"]["extensions"]): # the process must be restarted with updated config raise RuntimeError("RESTART: differences in extensions:" diff --git a/experiment/requirements.txt b/experiment/requirements.txt index a1aaf54c9..c4fca9754 100644 --- a/experiment/requirements.txt +++ b/experiment/requirements.txt @@ -1,16 +1,16 @@ -# Python 3.10.19 -# pip 25.2 +# Python 3.11.14 +# pip 25.3 # version sensetive -h5py==3.12.1 -keras==2.15.0 +h5py==3.14.0 +keras==3.14.0 keras-tuner==1.4.7 numpy==1.26.4 onnx==1.17.0 -protobuf==3.20.3 +protobuf==7.34.1 scikit-learn==1.6.1 -tensorflow-cpu==2.15.1 -tf2onnx==1.16.1 +tensorflow-cpu==2.21.0 +tf2onnx==1.17.0 wrapt==1.14.1 # version insensetive diff --git a/experiment/train.py b/experiment/train.py index e029e4f26..f160e22f1 100644 --- a/experiment/train.py +++ b/experiment/train.py @@ -143,7 +143,7 @@ def train( directory=str(RESULTS_DIR / f"{current_time}.tuner"), project_name='ml_tuning', seed=random.randint(1, 0xffffffff), - max_trials=30, + max_trials=3, ) search_early_stopping = EarlyStopping(monitor="val_loss", patience=patience, @@ -187,7 +187,7 @@ def train( mode="min", restore_best_weights=True, verbose=1) - model_checkpoint = ModelCheckpoint(filepath=str(RESULTS_DIR / f"{current_time}.best_model"), + model_checkpoint = ModelCheckpoint(filepath=str(RESULTS_DIR / f"{current_time}.best_model.keras"), monitor="val_loss", save_best_only=True, mode="min", @@ -195,16 +195,16 @@ def train( print(f"Memory before train: {LogCallback.get_memory_info()}", flush=True) - fit_history = keras_model.fit(x=[x_train_line, x_train_variable, x_train_value, x_train_features], - y=y_train, - batch_size=batch_size, - epochs=epochs, - verbose=2, - validation_data=([x_test_line, x_test_variable, x_test_value, - x_test_features], y_test), - class_weight=class_weight, - callbacks=[early_stopping, model_checkpoint, log_callback], - use_multiprocessing=True) + fit_history = keras_model.fit( + x=[x_train_line, x_train_variable, x_train_value, x_train_features], + y=y_train, + batch_size=batch_size, + epochs=epochs, + verbose=2, + validation_data=([x_test_line, x_test_variable, x_test_value, x_test_features], y_test), + class_weight=class_weight, + callbacks=[early_stopping, model_checkpoint, log_callback], + ) # if best_val_loss is not None and best_val_loss + 0.00001 < early_stopping.best: # print(f"CHECK BEST TUNER EARLY STOP : {best_val_loss} vs CURRENT: {early_stopping.best}",flush=True) @@ -215,7 +215,7 @@ def train( pickle.dump(fit_history, f) model_file_name = RESULTS_DIR / f"ml_model_at-{current_time}" - keras_model.save(model_file_name, include_optimizer=False) + keras_model.export(model_file_name, verbose=True, include_optimizer=False) if eval_test: print(f"Validate results on the test subset. Size: {len(y_test)} {np.mean(y_test):.4f}", flush=True) diff --git a/fuzz/corpus/oom-dbae986021f67d53fa03f642868df1db40fe6dc2 b/fuzz/corpus/oom-dbae986021f67d53fa03f642868df1db40fe6dc2 new file mode 100644 index 000000000..213b2c583 Binary files /dev/null and b/fuzz/corpus/oom-dbae986021f67d53fa03f642868df1db40fe6dc2 differ diff --git a/fuzz/requirements.txt b/fuzz/requirements.txt index 85631dccb..3ca42e0ca 100644 --- a/fuzz/requirements.txt +++ b/fuzz/requirements.txt @@ -1,3 +1,6 @@ +# Python 3.11.14 +# pip 25.3 + # Fuzzing atheris==2.3.0; python_version < '3.12' atheris==3.0.0; python_version >= '3.12' and python_version < '3.14' diff --git a/pyproject.toml b/pyproject.toml index 1e62a329a..5e852b335 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "whatthepatch", "xlrd", ] -requires-python = ">=3.10" +requires-python = ">=3.11" readme = "README.md" license = {text = "MIT"} classifiers = [ @@ -37,7 +37,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/requirements.txt b/requirements.txt index 5fbbedd4f..697227e04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# Python 3.10.20 +# Python 3.11.15 # pip 26.0.1 # build requirement @@ -17,22 +17,19 @@ cryptography==47.0.0 GitPython==3.1.47 humanfriendly==10.0 lxml==6.1.0 -numpy==2.2.6; python_version == '3.10' -numpy==2.4.4; python_version > '3.10' +numpy==2.4.4 odfpy==1.4.1 xlrd==2.0.2 striprtf==0.0.32 # onnxruntime - ML engine -onnxruntime==1.23.2; python_version < '3.14' -onnxruntime==1.24.4; python_version >= '3.14' +onnxruntime==1.25.1 # openpyxl - A Python library to read/write Excel 2010 xlsx/xlsm files openpyxl==3.1.5 # pandas - ML requirement and excel data reading -pandas==2.3.3; python_version == '3.10' -pandas==3.0.2; python_version > '3.10' +pandas==3.0.2 pdfminer.six==20260107 pybase62==1.0.0 @@ -53,13 +50,16 @@ pytest pytest-cov pytest-html pytest-random-order + # linters pylint mypy yapf + # stubs lxml-stubs pandas-stubs + # types types-beautifulsoup4 types-colorama diff --git a/tests/test_app.py b/tests/test_app.py index ad215c18f..b9a188724 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -294,8 +294,6 @@ def test_log_n(self) -> None: # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - # todo: fix when python 3.10 support ends - @pytest.mark.skipif(10 < sys.version_info.minor, reason="argparse default was changed in 3.11") def test_help_p(self) -> None: _stdout, _stderr = self._m_credsweeper(["--help"]) output = " ".join(_stdout.split())