[AP-2963] Updates for Python 3.12 which is default in Ubuntu 24

.github/workflows/connectors.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   test_install_connectors:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checking out repo
@@ -27,10 +27,10 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5.1.0
         with:
-          python-version: '3.10'
+          python-version: '3.12'
 
       - name: Check PipelineWise and all connectors are installable
         run: |

.github/workflows/e2e_tests.yml

@@ -29,7 +29,7 @@ env:
 
 jobs:
   e2e_tests_target_pg:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     environment: ci_tests
 
     steps:
@@ -70,7 +70,7 @@ jobs:
           pipelinewise_dev pytest tests/end_to_end/test_target_postgres.py -vx --timer-top-n 10
 
   e2e_tests_mariadb_to_sf:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     environment: ci_tests
 
     steps:
@@ -122,7 +122,7 @@ jobs:
           pipelinewise_dev pytest tests/end_to_end/target_snowflake/tap_mariadb -vx --timer-top-n 10
 
   e2e_tests_pg_to_sf:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     environment: ci_tests
 
     steps:
@@ -174,7 +174,7 @@ jobs:
           pipelinewise_dev pytest tests/end_to_end/target_snowflake/tap_postgres -vx --timer-top-n 10
 
   e2e_tests_mg_to_sf:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     environment: ci_tests
 
     steps:
@@ -226,7 +226,7 @@ jobs:
           pipelinewise_dev pytest tests/end_to_end/target_snowflake/tap_mongodb -vx --timer-top-n 10
 
   e2e_tests_s3_to_sf:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     environment: ci_tests
 
     steps:

.github/workflows/lint_unit_tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   lint_and_test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checking out repo
@@ -26,11 +26,11 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v5.1.0
         with:
-          python-version: '3.10'
+          python-version: '3.12'
 
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
@@ -40,7 +40,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           . .virtualenvs/pipelinewise/bin/activate
-          find pipelinewise tests -type f -name '*.py' | xargs unify --check-only
+          ruff check pipelinewise tests
 
       - name: Pylinting
         if: steps.check.outcome == 'failure'

.github/workflows/publish_doc.yml

@@ -23,7 +23,7 @@ jobs:
 
       - uses: actions/setup-python@v5.1.0
         with:
-          python-version: '3.10'
+          python-version: '3.12'
 
       - name: check structure
         run: ls -l gh_doc_automation

CHANGELOG.md

@@ -1,3 +1,7 @@
+0.76.0 (2026-04-16)
+-------------------
+- Updates for Python 3.12 
+
 0.75.0 (2026-04-07)
 -------------------
 - `pipelinewise-tap-postgres` from `2.1.0` to `2.2.0`

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim-bullseye
+FROM python:3.12-slim-bullseye
 
 ARG connectors=all
 
@@ -15,12 +15,13 @@ RUN apt-get -qq update \
     && pip install -U --no-cache-dir pip
 
 # Add Mongodb ppa
-RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \
-    && echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb.list \
-    && apt-get -qq update \
-    && apt-get -qqy --no-install-recommends install \
-        mongodb-database-tools \
-    && rm -rf /var/lib/apt/lists/*
+RUN ARCH=$(dpkg --print-architecture) && \
+    wget -qO- https://www.mongodb.org/static/pgp/server-4.4.asc | gpg --dearmor > /usr/share/keyrings/mongodb-archive-keyring.gpg && \
+    echo "deb [ arch=${ARCH} signed-by=/usr/share/keyrings/mongodb-archive-keyring.gpg ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" > /etc/apt/sources.list.d/mongodb.list && \
+    apt-get -qq update && \
+    apt-get -qqy --no-install-recommends install \
+        mongodb-database-tools && \
+    rm -rf /var/lib/apt/lists/*
 
 COPY singer-connectors/ /app/singer-connectors/
 COPY Makefile /app

Dockerfile.barebone

@@ -1,4 +1,4 @@
-FROM python:3.10-slim-bullseye
+FROM python:3.12-slim-bullseye
 
 RUN apt-get -qq update \
     && apt-get -qqy --no-install-recommends install \

dev-project/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   ### Primary container with PipelineWise CLI
   pipelinewise:
     platform: linux/amd64
-    image: ubuntu:22.04
+    image: ubuntu:24.04
     container_name: pipelinewise_dev
     working_dir: /opt/pipelinewise
     entrypoint: /opt/pipelinewise/dev-project/entrypoint.sh

dev-project/entrypoint.sh

@@ -20,7 +20,7 @@ apt_retry() {
 }
 
 apt-get update
-apt_retry apt-get install -y software-properties-common python3-apt apt-utils
+apt_retry apt-get install -y software-properties-common apt-utils
 
 add-apt-repository ppa:deadsnakes/ppa
 apt-get update
@@ -33,14 +33,11 @@ apt_retry apt-get install -y --no-install-recommends \
   git \
   alien \
   gettext-base \
-  libaio1 \
+  libaio1t64 \
   mariadb-client \
   mbuffer \
   postgresql-client \
-  python3.10 python3-pip python3.10-venv python3.10-dev
-
-rm /usr/bin/python3
-ln -s /usr/bin/python3.10 /usr/bin/python3
+  python3.12-dev python3.12-venv
 
 apt-get upgrade -y
 

pipelinewise/cli/__init__.py

@@ -265,7 +265,7 @@ def main():
     # import_config : this is for backward compatibility; use 'import' instead from CLI
     # Every command argument is mapped to a python function with the same name, but 'import' is a
     # python keyword and can't be used as function name
-    if args.command == 'import' or args.command == 'import_config':
+    if args.command in ['import', 'import_config']:
         args.command = 'import_project'
     try:
         _validate_command_specific_arguments(args)

pipelinewise/cli/alert_handlers/victorops_alert_handler.py

@@ -54,6 +54,7 @@ def send(
             Initialised alert handler object
         """
         # Send alert to VictorOps REST Endpoint as a HTTP post request
+        # pylint: disable=missing-timeout
         response = requests.post(
             f'{self.base_url}/{self.routing_key}',
             data=json.dumps(

pipelinewise/cli/commands.py

@@ -149,7 +149,7 @@ def exists_and_executable(bin_path: str) -> bool:
 
         try:
             paths = f"{os.environ['PATH']}".split(':')
-            (p for p in paths if os.access(f'{p}/{bin_path}', os.X_OK)).__next__()
+            next(p for p in paths if os.access(f'{p}/{bin_path}', os.X_OK))
         except StopIteration:
             return False
     return True
@@ -296,6 +296,7 @@ def build_stream_buffer_command(
     return buffer_command
 
 
+# pylint: disable=too-many-positional-arguments
 def build_singer_command(
     tap: TapParams,
     target: TargetParams,
@@ -354,6 +355,7 @@ def build_singer_command(
     return command
 
 
+# pylint: disable=too-many-positional-arguments
 # pylint: disable=too-many-arguments
 def build_partialsync_command(
         tap: TapParams,
@@ -400,6 +402,7 @@ def build_partialsync_command(
     return command
 
 
+# pylint: disable=too-many-positional-arguments
 # pylint: disable=too-many-arguments
 def build_fastsync_command(
     tap: TapParams,

pipelinewise/cli/pipelinewise.py

@@ -166,6 +166,7 @@ def create_consumable_target_config(self, target_config, tap_inheritable_config)
                 f'Cannot merge JSON files {dict_a} {dict_b} - {exc}'
             ) from exc
 
+    # pylint: disable=too-many-positional-arguments
     # pylint: disable=too-many-statements,too-many-branches,too-many-nested-blocks,too-many-locals,too-many-arguments
     def create_filtered_tap_properties(
         self,
@@ -2273,6 +2274,7 @@ def cleanup_after_deleted_config(self, old_config: Dict) -> int:
 
         deleted_taps_count = 0
         for target_id, taps in old_config_dict.items():
+            # pylint: disable=unreachable
             if target_id not in new_config_dict:
                 # target is no longer configured, thus we need to remove all its config and taps tied to it
                 self._remove_target_config(target_id, taps)

pipelinewise/fastsync/commons/tap_mongodb.py

@@ -235,7 +235,7 @@ def close_connection(self):
         """
         self.database.client.close()
 
-    # pylint: disable=R0914,R0913
+    # pylint: disable=R0914,R0913,R0917
     def copy_table(
         self,
         table_name: str,

pipelinewise/fastsync/commons/tap_mysql.py

@@ -347,6 +347,7 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
         schema_name = table_dict.get('schema_name')
         table_name = table_dict.get('table_name')
 
+        # pylint: disable=line-too-long
         sql = f"""
                 SELECT column_name AS column_name,
                     data_type AS data_type,
@@ -385,6 +386,7 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
                 ORDER BY
                         ordinal_position
             """  # noqa: E501
+        # pylint: enable=line-too-long
         return self.query(sql)
 
     def map_column_types_to_target(self, table_name):
@@ -407,7 +409,7 @@ def map_column_types_to_target(self, table_name):
             'primary_key': self.get_primary_keys(table_name),
         }
 
-    # pylint: disable=too-many-locals
+    # pylint: disable=too-many-locals, too-many-positional-arguments
     def copy_table(
             self,
             table_name,

pipelinewise/fastsync/commons/tap_postgres.py

@@ -271,9 +271,11 @@ def fetch_current_log_pos(self):
         self.primary_host_curr = self.primary_host_conn.cursor()
 
         # Make sure PostgreSQL version is 9.4 or higher
+        # pylint: disable=assignment-from-no-return
         result = self.primary_host_query(
             "SELECT setting::int AS version FROM pg_settings WHERE name='server_version_num'"
         )
+        # pylint: disable=unsubscriptable-object
         version = result[0].get('version')
 
         # Do not allow minor versions with PostgreSQL BUG #15114
@@ -395,9 +397,10 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
 
         if max_num:
             decimals = len(max_num.split('.')[1]) if '.' in max_num else 0
+
             decimal_format = f"""
               'CASE WHEN "' || column_name || '" IS NULL THEN NULL ELSE GREATEST(LEAST({max_num}, ROUND("' || column_name || '"::numeric , {decimals})), -{max_num}) END'
-            """ # noqa E501
+            """ # noqa E501 pylint: disable=line-too-long
             integer_format = """
               '"' || column_name || '"'
             """
@@ -410,6 +413,7 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
         schema_name = table_dict.get('schema_name')
         table_name = table_dict.get('table_name')
 
+        # pylint: disable = line-too-long
         sql = f"""
                 SELECT
                     column_name
@@ -442,6 +446,8 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
                 ORDER BY ordinal_position
                 ) AS x
             """  # noqa: E501
+        # pylint: enable = line-too-long
+
         return self.query(sql)
 
     def map_column_types_to_target(self, table_name):
@@ -466,7 +472,7 @@ def map_column_types_to_target(self, table_name):
             'primary_key': self.get_primary_keys(table_name),
         }
 
-    # pylint: disable=too-many-arguments, too-many-locals
+    # pylint: disable=too-many-arguments, too-many-locals, too-many-positional-arguments
     def copy_table(
         self,
         table_name,

pipelinewise/fastsync/commons/target_postgres.py

@@ -12,7 +12,7 @@
 LOGGER = logging.getLogger(__name__)
 
 
-# pylint: disable=missing-function-docstring,no-self-use,too-many-arguments
+# pylint: disable=missing-function-docstring,too-many-arguments
 class FastSyncTargetPostgres:
     """
     Common functions for fastsync to Postgres

pipelinewise/fastsync/commons/target_snowflake.py

@@ -18,7 +18,7 @@
 logging.getLogger('snowflake.connector').setLevel(logging.WARNING)
 
 
-# pylint: disable=missing-function-docstring,no-self-use,too-many-arguments
+# pylint: disable=missing-function-docstring,too-many-arguments
 class FastSyncTargetSnowflake:
     """
     Common functions for fastsync to Snowflake
@@ -214,6 +214,7 @@ def drop_table(self, target_schema, table_name, is_temporary=False):
         sql = 'DROP TABLE IF EXISTS {}."{}"'.format(target_schema, target_table.upper())
         self.query(sql, query_tag_props={'schema': target_schema, 'table': table_name})
 
+    # pylint: disable=too-many-positional-arguments
     def create_table(
         self,
         target_schema: str,

pipelinewise/fastsync/partialsync/utils.py

@@ -158,7 +158,7 @@ def _validate_dynamic_boundary_value(query_object, string_to_check: str) -> str:
         else:
             boundary_value = return_value[0][0]
     except Exception:
-        raise(InvalidConfigException(f'Invalid query for boundary value: {string_to_check}')) from Exception
+        raise (InvalidConfigException(f'Invalid query for boundary value: {string_to_check}')) from Exception
     return boundary_value