diff --git a/.github/workflows/dev-build-to-pypi.yml b/.github/workflows/dev-build-to-pypi.yml index a4c5b88..d0120e2 100644 --- a/.github/workflows/dev-build-to-pypi.yml +++ b/.github/workflows/dev-build-to-pypi.yml @@ -35,14 +35,19 @@ jobs: filename: './package_info.json' key: 'package_version' value: ${{ steps.release_ver.outputs.value }} - # Build and publish + # Test - name: Set up Python 3.7 uses: actions/setup-python@v1 with: python-version: 3.7 - - name: Build package + - name: Run tests run: | pip install -r requirements.txt + pip install pytest + pytest + # Build and publish + - name: Build package + run: | python setup.py sdist # - name: Publish distribution to Test PyPI # uses: pypa/gh-action-pypi-publish@master diff --git a/.github/workflows/mypy-flake8-checks.yaml b/.github/workflows/mypy-flake8-checks.yaml new file mode 100644 index 0000000..9f6a964 --- /dev/null +++ b/.github/workflows/mypy-flake8-checks.yaml @@ -0,0 +1,50 @@ +name: MyPy & flake8 checks + +on: + push: + +jobs: + test: + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.sha }} + - name: Install Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + + pip install mypy + python3 -m pip install types-protobuf + + flake8_plugins=" + flake8 + flake8-bugbear + flake8-builtins + flake8-comprehensions + flake8-eradicate + flake8-fixme + flake8-multiline-containers + flake8-print + flake8-return + flake8-quotes + flake8-simplify + pep8-naming + flake8-expression-complexity + flake8-import-order + flake8-annotations-complexity + flake8-annotations-coverage + " + pip install $flake8_plugins + - name: Run MyPy + continue-on-error: true + run: mypy . + - name: Run flake8 + run: flake8 . \ No newline at end of file diff --git a/.gitignore b/.gitignore index 142c22d..6984d72 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ __pycache__/ *.py[cod] *$py.class dist -*egg-info \ No newline at end of file +*egg-info +.mypy_cache +all.log \ No newline at end of file diff --git a/package_info.json b/package_info.json index 451a314..c2c2c31 100644 --- a/package_info.json +++ b/package_info.json @@ -1,4 +1,4 @@ { "package_name": "th2-common", - "package_version": "3.8.2" + "package_version": "3.9.1" } diff --git a/py.typed b/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index f2afd15..8e23618 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,7 @@ # limitations under the License. aio_pika==6.8.2 -uvloop==0.16.0 -th2-grpc-common==3.9.0 -kubernetes==18.20.0 -prometheus_client==0.11.0 -th2-common-utils==1.1.0 \ No newline at end of file +th2-grpc-common==3.11.1.dev2999890198 +kubernetes==24.2.0 +prometheus_client==0.14.1 +th2-common-utils==1.4.3 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..68133a2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,58 @@ +[mypy] +disallow_untyped_defs = True +ignore_missing_imports = True +no_implicit_optional = True +warn_return_any = True + +[flake8] +filename = *.py +require-plugins = +; Finding likely bugs and design problems in your program. + flake8-bugbear +; Check for python builtins being used as variables or parameters. + flake8-builtins +; Helps you write better list/set/dict comprehensions. + flake8-comprehensions +; Plugin to find commented out or dead code. + flake8-eradicate +; Check for FIXME, TODO and other temporary developer notes. + flake8-fixme +; Plugin to ensure a consistent format for multiline containers. + flake8-multiline-containers +; Check for print statements in python files. + flake8-print +; Plugin that checks return values. + flake8-return +; Extension for checking quotes in python. + flake8-quotes +; Plugin that helps you to simplify code. + flake8-simplify +; Check the PEP-8 naming conventions. + pep8-naming +; Extension for flake8 that validates cognitive functions complexity. +; flake8-cognitive-complexity +; Plugin to validate expressions complexity. + flake8-expression-complexity +; Include checks import order against various Python Style Guides. + flake8-import-order +; Plugin to validate annotations complexity. + flake8-annotations-complexity +; Plugin to validate annotations coverage. + flake8-annotations-coverage +ignore = N803, N806, T100, B010, E126, B024 +per-file-ignores = + th2_common/__init__.py: F403, F401, T201 + test/conftest.py: F401 + +import-order-style = google +max-line-length = 120 +max-doc-length = 120 +indent-size = 2 +inline-quotes = ' +multiline-quotes = ''' +docstring-quotes = """ + +max-cognitive-complexity = 8 + +count = True +statistics = True diff --git a/setup.py b/setup.py index 970ab69..77d6016 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ import json -from setuptools import setup, find_packages +from setuptools import find_packages, setup with open('package_info.json', 'r') as file: package_info = json.load(file) @@ -38,12 +38,11 @@ python_requires='>=3.7', install_requires=[ 'aio_pika==6.8.2', - 'uvloop==0.16.0', - 'th2-grpc-common~=3.9.0', - 'kubernetes==18.20.0', - 'prometheus_client==0.11.0', - 'th2-common-utils==1.1.0' + 'th2-grpc-common==3.11.1.dev2999890198', + 'kubernetes==24.2.0', + 'prometheus_client==0.14.1', + 'th2-common-utils>=1.4.3' ], packages=[''] + find_packages(include=['th2_common', 'th2_common.*']), - package_data={'': ['package_info.json'], 'th2_common.schema.log': ['log4py.conf']} + package_data={'': ['package_info.json'], 'th2_common.schema.log': ['log4py.conf', 'log_config.json']} ) diff --git a/th2_common/schema/strategy/route/impl/__init__.py b/test/__init__.py similarity index 90% rename from th2_common/schema/strategy/route/impl/__init__.py rename to test/__init__.py index 223b64c..9f4cc62 100644 --- a/th2_common/schema/strategy/route/impl/__init__.py +++ b/test/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/th2_common/schema/strategy/route/routing_strategy.py b/test/conftest.py similarity index 69% rename from th2_common/schema/strategy/route/routing_strategy.py rename to test/conftest.py index a6802a6..4e68e17 100644 --- a/th2_common/schema/strategy/route/routing_strategy.py +++ b/test/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. +from test.fixtures import factory, filtered_messages, grpc_router -from abc import ABC, abstractmethod - -from google.protobuf.message import Message - - -class RoutingStrategy(ABC): - - @abstractmethod - def get_endpoint(self, message: Message): - pass +import pytest diff --git a/test/fixtures.py b/test/fixtures.py new file mode 100644 index 0000000..e9a9080 --- /dev/null +++ b/test/fixtures.py @@ -0,0 +1,56 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from test.test_filter_strategy.resources.messages_for_filtering import message_group_batch +from typing import Any, Dict +from unittest.mock import Mock + +import pytest +from th2_common.schema.factory.common_factory import CommonFactory +from th2_common.schema.grpc.router.impl.default_grpc_router import DefaultGrpcRouter +from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router import \ + RabbitMessageGroupBatchRouter + + +@pytest.fixture(scope='session') +def factory() -> CommonFactory: # type: ignore + config_path = Path(__file__).parent / 'test_configuration' / 'resources' / 'json_configuration' + factory = CommonFactory(config_path=config_path) + + yield factory + + factory.close() + + +@pytest.fixture(scope='session') +def filtered_messages(factory) -> Dict[str, Any]: # type: ignore + message_router_configuration = factory._create_message_router_configuration() + rabbit_message_router = RabbitMessageGroupBatchRouter(connection_manager=Mock(), + configuration=message_router_configuration) + + filtered_messages = rabbit_message_router.split_and_filter( + queue_aliases_to_configs=message_router_configuration.queues, + batch=message_group_batch + ) + + yield filtered_messages + + +@pytest.fixture(scope='session') +def grpc_router(factory) -> DefaultGrpcRouter: # type: ignore + grpc_configuration = factory._create_grpc_configuration() + grpc_router = DefaultGrpcRouter(grpc_configuration=grpc_configuration, grpc_router_configuration=Mock()) + + yield grpc_router diff --git a/th2_common/schema/strategy/route/__init__.py b/test/test_configuration/__init__.py similarity index 90% rename from th2_common/schema/strategy/route/__init__.py rename to test/test_configuration/__init__.py index 223b64c..9f4cc62 100644 --- a/th2_common/schema/strategy/route/__init__.py +++ b/test/test_configuration/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/test_configuration/resources/dict_configuration.py b/test/test_configuration/resources/dict_configuration.py new file mode 100644 index 0000000..3bfde6f --- /dev/null +++ b/test/test_configuration/resources/dict_configuration.py @@ -0,0 +1,298 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +rabbit_mq_configuration_dict = { + 'exchange_name': 'exchange', + 'host': '0.0.0.0', + 'password': 'password', + 'port': '10', + 'prefetch_count': 100, + 'subscriber_name': None, + 'username': 'name', + 'vhost': 'v-host' +} + +mq_configuration_dict = { # noqa: ECE001 + 'queues': { + 'queue_AND_filter': { + 'attributes': ['attr11', 'attr12'], + 'can_read': True, + 'can_write': True, + 'exchange': 'exchange1', + 'filters': [ + { + 'message': [ + { + 'field_name': 'msg11', + 'operation': 'EQUAL', + 'value': '11' + } + ], + 'metadata': [ + { + 'field_name': 'session_alias', + 'operation': 'EQUAL', + 'value': 'qwerty' + } + ] + } + ], + 'queue': '', + 'routing_key': 'name1' + }, + 'queue_EMPTY_filter': { + 'attributes': [], + 'can_read': True, + 'can_write': True, + 'exchange': 'exchange3', + 'filters': [], + 'queue': 'queue_EMPTY_filter', + 'routing_key': 'name3' + }, + 'queue_OR_AND_filter': { + 'attributes': [], + 'can_read': True, + 'can_write': True, + 'exchange': 'exchange3', + 'filters': [ + { + 'message': [], + 'metadata': [ + { + 'field_name': 'session_alias', + 'operation': 'EQUAL', + 'value': 'asdfgh' + }, + { + 'field_name': 'direction', + 'operation': 'EQUAL', + 'value': 'SECOND' + } + ] + }, + { + 'message': [ + { + 'field_name': 'msg31', + 'operation': 'EQUAL', + 'value': '31' + }, + { + 'field_name': 'msg32', + 'operation': 'EQUAL', + 'value': '32' + } + ], + 'metadata': [] + } + ], + 'queue': '', + 'routing_key': 'name3' + }, + 'queue_OR_filter': { + 'attributes': ['attr21', 'attr22', 'attr23'], + 'can_read': True, + 'can_write': True, + 'exchange': 'exchange2', + 'filters': [ + { + 'message': [ + { + 'field_name': 'msg21', + 'operation': 'EQUAL', + 'value': '21' + } + ], + 'metadata': [] + }, + { + 'message': [ + { + 'field_name': 'msg22', + 'operation': 'EQUAL', + 'value': '22' + } + ], + 'metadata': [] + } + ], + 'queue': 'queue_OR_filter', + 'routing_key': '' + } + } +} + +mq_router_configuration_dict = { + 'connection_close_timeout': 10, + 'connection_timeout': -1, + 'max_connection_recovery_timeout': 6000, + 'max_recovery_attempts': 5, + 'message_recursion_limit': 100, + 'min_connection_recovery_timeout': 10000, + 'prefetch_count': 50, + 'subscriber_name': None +} + +grpc_configuration_dict = { # noqa: ECE001 + 'server': { + 'attributes': None, + 'host': 'localhost', + 'port': 8080, + 'workers': 5 + }, + 'services': { + 'service1': { + 'attributes': [], + 'endpoints': { + 'endpoint11': { + 'attributes': [], + 'host': '0.0.0.1', + 'port': '1010' + }, + 'endpoint12': { + 'attributes': [], + 'host': '0.0.0.2', + 'port': 1001 + } + }, + 'filters': [ + { + 'properties': [ + { + 'field_name': 'session_alias', + 'operation': 'NOT_EQUAL', + 'value': 'qwerty' + }, + { + 'field_name': 'msg11', + 'operation': 'EMPTY', + 'value': None + } + ] + } + ], + 'service_class': 'ServiceClass1' + }, + 'service2': { + 'attributes': [], + 'endpoints': { + 'endpoint2.1': { + 'attributes': [], + 'host': '0.0.0.3', + 'port': 1011 + } + }, + 'filters': [ + { + 'properties': [ + { + 'field_name': 'prop21', + 'operation': 'EQUAL', + 'value': '21' + }, + { + 'field_name': 'prop22', + 'operation': 'EQUAL', + 'value': '22' + } + ] + }, + { + 'properties': [ + { + 'field_name': 'prop23', + 'operation': 'NOT_EMPTY', + 'value': None + } + ] + } + ], + 'service_class': 'ServiceClass2' + }, + 'service3': { + 'attributes': [], + 'endpoints': {}, + 'filters': [ + { + 'properties': [ + { + 'field_name': 'prop31', + 'operation': 'EQUAL', + 'value': '31' + }, + { + 'field_name': 'prop32', + 'operation': 'EQUAL', + 'value': '32' + } + ] + } + ], + 'service_class': 'ServiceClass3' + }, + 'service4': { # the same as service3 + 'attributes': [], + 'endpoints': {}, + 'filters': [ + { + 'properties': [ + { + 'field_name': 'prop31', + 'operation': 'EQUAL', + 'value': '31' + }, + { + 'field_name': 'prop32', + 'operation': 'EQUAL', + 'value': '32' + } + ] + } + ], + 'service_class': 'ServiceClass3' + } + } +} + +grpc_router_configuration_dict = { + 'retry_policy': { + 'backoff_multiplier': 2, + 'initial_backoff': 5.0, + 'max_attempts': 5, + 'max_backoff': 60.0, + 'services': None, + 'status_codes': None, + }, + 'request_size_limit': [ + ('grpc.max_receive_message_length', 4194304), + ('grpc.max_send_message_length', 4194304) + ], + 'workers': 5 +} + +cradle_configuration_dict = { + 'data_center': 'some_datacenter', + 'host': 'localhost', + 'port': 8080, + 'keyspace': 'some_keyspace', + 'username': 'user1', + 'password': 'password123' +} + +prometheus_configuration_dict = { + 'enabled': False, + 'host': '0.0.0.0', + 'port': 9752 +} diff --git a/test/test_configuration/resources/json_configuration/cradle.json b/test/test_configuration/resources/json_configuration/cradle.json new file mode 100644 index 0000000..23c34d8 --- /dev/null +++ b/test/test_configuration/resources/json_configuration/cradle.json @@ -0,0 +1,8 @@ +{ + "dataCenter":"some_datacenter", + "host":"localhost", + "port":8080, + "keyspace":"some_keyspace", + "username":"user1", + "password":"password123" +} \ No newline at end of file diff --git a/test/test_configuration/resources/json_configuration/grpc.json b/test/test_configuration/resources/json_configuration/grpc.json new file mode 100644 index 0000000..9e41cc7 --- /dev/null +++ b/test/test_configuration/resources/json_configuration/grpc.json @@ -0,0 +1,123 @@ +{ + "server":{ + "attributes":null, + "host":"localhost", + "port":8080, + "workers":5 + }, + + "services": { + "service1": { + "endpoints": { + "endpoint11": { + "attributes": [], + "host": "0.0.0.1", + "port": "1010" + }, + "endpoint12": { + "attributes": [], + "host": "0.0.0.2", + "port": 1001 + } + }, + "service-class": "ServiceClass1", + "strategy": { + "endpoints": [ + "endpoint" + ], + "name": "robin" + }, + "filters": [ + { + "metadata": [ + { + "fieldName": "session_alias", + "expectedValue": "qwerty", + "operation": "NOT_EQUAL" + } + ], + "message": [ + { + "fieldName": "msg11", + "operation": "EMPTY" + } + ] + } + ] + }, + "service2": { + "endpoints": { + "endpoint2.1": { + "attributes": [], + "host": "0.0.0.3", + "port": 1011 + } + }, + "service-class": "ServiceClass2", + "filters": [ + { + "properties": [ + { + "fieldName": "prop21", + "value": "21", + "operation": "EQUAL" + }, + { + "fieldName": "prop22", + "value": "22", + "operation": "EQUAL" + } + ] + }, + { + "properties": [ + { + "fieldName": "prop23", + "operation": "NOT_EMPTY" + } + ] + } + ] + }, + "service3": { + "endpoints": {}, + "service-class": "ServiceClass3", + "filters": [ + { + "properties": [ + { + "fieldName": "prop31", + "value": "31", + "operation": "EQUAL" + }, + { + "fieldName": "prop32", + "value": "32", + "operation": "EQUAL" + } + ] + } + ] + }, + "service4": { + "endpoints": {}, + "service-class": "ServiceClass3", + "filters": [ + { + "properties": [ + { + "fieldName": "prop31", + "value": "31", + "operation": "EQUAL" + }, + { + "fieldName": "prop32", + "value": "32", + "operation": "EQUAL" + } + ] + } + ] + } + } +} diff --git a/test/test_configuration/resources/json_configuration/grpc_router.json b/test/test_configuration/resources/json_configuration/grpc_router.json new file mode 100644 index 0000000..4f2c1e4 --- /dev/null +++ b/test/test_configuration/resources/json_configuration/grpc_router.json @@ -0,0 +1,7 @@ +{ + "workers": 5, + "options": { + "option1": "value1", + "option2": 2 + } +} \ No newline at end of file diff --git a/test/test_configuration/resources/json_configuration/mq.json b/test/test_configuration/resources/json_configuration/mq.json new file mode 100644 index 0000000..9043f62 --- /dev/null +++ b/test/test_configuration/resources/json_configuration/mq.json @@ -0,0 +1,97 @@ +{ + "queues": { + "queue_AND_filter":{ + "attributes":["attr11","attr12"], + "exchange":"exchange1", + "filters":[ + { + "metadata":[ + { + "value":"qwerty", + "fieldName":"session_alias", + "operation":"EQUAL" + } + ], + "message":[ + { + "value":"11", + "fieldName":"msg11", + "operation":"EQUAL" + } + ] + } + ], + "name":"name1", + "queue":"" + }, + "queue_OR_filter":{ + "attributes":["attr21","attr22","attr23"], + "exchange":"exchange2", + "filters":[ + { + "message":[ + { + "value":"21", + "fieldName":"msg21", + "operation":"EQUAL" + } + ] + }, + { + "message":[ + { + "value":"22", + "fieldName":"msg22", + "operation":"EQUAL" + } + ] + } + ], + "name":"", + "queue":"queue_OR_filter" + }, + "queue_OR_AND_filter":{ + "attributes":[], + "exchange":"exchange3", + "filters": [ + { + "metadata":[ + { + "expectedValue":"asdfgh", + "fieldName":"session_alias", + "operation":"EQUAL" + }, + { + "expectedValue":"SECOND", + "fieldName":"direction", + "operation":"EQUAL" + } + ] + }, + { + "message":[ + { + "value":"31", + "fieldName":"msg31", + "operation":"EQUAL" + }, + { + "expectedValue":"32", + "fieldName":"msg32", + "operation":"EQUAL" + } + ] + } + ], + "name":"name3", + "queue":"" + }, + "queue_EMPTY_filter":{ + "attributes":[], + "exchange":"exchange3", + "filters": [], + "name":"name3", + "queue":"queue_EMPTY_filter" + } + } +} \ No newline at end of file diff --git a/test/test_configuration/resources/json_configuration/mq_router.json b/test/test_configuration/resources/json_configuration/mq_router.json new file mode 100644 index 0000000..f473ded --- /dev/null +++ b/test/test_configuration/resources/json_configuration/mq_router.json @@ -0,0 +1,8 @@ +{ + "connectionCloseTimeout":10, + "connectionTimeout":"-1", + "maxConnectionRecoveryTimeout":6000, + "maxRecoveryAttempts":"5", + "minConnectionRecoveryTimeout":"10000", + "prefetchCount":50 +} \ No newline at end of file diff --git a/test/test_configuration/resources/json_configuration/prometheus.json b/test/test_configuration/resources/json_configuration/prometheus.json new file mode 100644 index 0000000..21f2adc --- /dev/null +++ b/test/test_configuration/resources/json_configuration/prometheus.json @@ -0,0 +1,3 @@ +{ + "enabled":false +} \ No newline at end of file diff --git a/test/test_configuration/resources/json_configuration/rabbitMQ.json b/test/test_configuration/resources/json_configuration/rabbitMQ.json new file mode 100644 index 0000000..73ace29 --- /dev/null +++ b/test/test_configuration/resources/json_configuration/rabbitMQ.json @@ -0,0 +1,8 @@ +{ + "host": "0.0.0.0", + "port": "10", + "vHost": "v-host", + "username": "name", + "password": "password", + "exchangeName": "exchange" +} diff --git a/test/test_configuration/test_json_configuration.py b/test/test_configuration/test_json_configuration.py new file mode 100644 index 0000000..a0d8462 --- /dev/null +++ b/test/test_configuration/test_json_configuration.py @@ -0,0 +1,85 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from test.test_configuration.resources import dict_configuration +from test.utils import object_to_dict + +import pytest +from th2_common.schema.factory.common_factory import CommonFactory + + +@pytest.mark.usefixtures('factory') +def test_rabbit_mq_configuration(factory: CommonFactory) -> None: + factory._create_rabbit_mq_configuration() + + expected = dict_configuration.rabbit_mq_configuration_dict + actual = factory.rabbit_mq_configuration + + assert object_to_dict(actual) == expected + + +@pytest.mark.usefixtures('factory') +def test_message_router_configuration(factory: CommonFactory) -> None: + factory._create_message_router_configuration() + + expected = dict_configuration.mq_configuration_dict + actual = factory.message_router_configuration + + assert object_to_dict(actual) == expected + + +@pytest.mark.usefixtures('factory') +def test_conn_manager_configuration(factory: CommonFactory) -> None: + expected = dict_configuration.mq_router_configuration_dict + actual = factory._create_conn_manager_configuration() + + assert object_to_dict(actual) == expected + + +@pytest.mark.usefixtures('factory') +def test_grpc_configuration(factory: CommonFactory) -> None: + factory._create_grpc_configuration() + + expected = dict_configuration.grpc_configuration_dict + actual = factory.grpc_configuration + + assert object_to_dict(actual) == expected + + +@pytest.mark.usefixtures('factory') +def test_grpc_router_configuration(factory: CommonFactory) -> None: + expected = dict_configuration.grpc_router_configuration_dict + actual = factory._create_grpc_router_configuration() + + assert object_to_dict(actual) == expected + + +@pytest.mark.usefixtures('factory') +def test_cradle_configuration(factory: CommonFactory) -> None: + factory.create_cradle_configuration() + + expected = dict_configuration.cradle_configuration_dict + actual = factory.create_cradle_configuration() + + assert object_to_dict(actual) == expected + + +@pytest.mark.usefixtures('factory') +def test_prometheus_configuration(factory: CommonFactory) -> None: + factory._create_prometheus_configuration() + + expected = dict_configuration.prometheus_configuration_dict + actual = factory.prometheus_config + + assert object_to_dict(actual) == expected diff --git a/th2_common/schema/strategy/field_extraction/impl/__init__.py b/test/test_filter_strategy/__init__.py similarity index 90% rename from th2_common/schema/strategy/field_extraction/impl/__init__.py rename to test/test_filter_strategy/__init__.py index 223b64c..9f4cc62 100644 --- a/th2_common/schema/strategy/field_extraction/impl/__init__.py +++ b/test/test_filter_strategy/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/test_filter_strategy/resources/messages_for_extraction.py b/test/test_filter_strategy/resources/messages_for_extraction.py new file mode 100644 index 0000000..17a319c --- /dev/null +++ b/test/test_filter_strategy/resources/messages_for_extraction.py @@ -0,0 +1,141 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import os +import random + +from google.protobuf.timestamp_pb2 import Timestamp +from th2_grpc_common.common_pb2 import AnyMessage, ByteMessage, ConnectionID, Direction, EventID, ListValue, \ + Message, MessageID, MessageMetadata, RawMessage, RawMessageMetadata, Value + +cl_ord_id = random.randint(10 ** 5, (10 ** 6) - 1) +transact_time = datetime.now().isoformat() +parent_event_id = EventID() +ts = Timestamp() + +# AnyMessage.message + +trading_party_message = Value(message_value=Message(fields={ + 'NoPartyIDs': Value(list_value=ListValue(values=[ + Value(message_value=Message(fields={ + 'PartyID': Value(simple_value='0'), + 'PartyIDSource': Value(simple_value='A'), + 'PartyRole': Value(simple_value='1') + })), + Value(message_value=Message(fields={ + 'PartyID': Value(simple_value='0'), + 'PartyIDSource': Value(simple_value='A'), + 'PartyRole': Value(simple_value='2') + })) + ])) +})) + +message = Message(parent_event_id=parent_event_id, + metadata=MessageMetadata( + message_type='NewOrderSingle', + id=MessageID( + connection_id=ConnectionID(session_alias='jsTUURfy'), + direction=Direction.SECOND + ) + ), + fields={ + 'OrdType': Value(simple_value='1'), + 'AccountType': Value(simple_value='2'), + 'OrderQty': Value(simple_value='3'), + 'Price': Value(simple_value='4'), + 'ClOrdID': Value(simple_value=str(cl_ord_id)), + 'TransactTime': Value(simple_value=transact_time), + 'TradingParty': trading_party_message + }) + +any_message = AnyMessage(message=message) + +trading_party_dict = { + 'NoPartyIDs': [ + { + 'PartyID': '0', + 'PartyIDSource': 'A', + 'PartyRole': '1' + }, + { + 'PartyID': '0', + 'PartyIDSource': 'A', + 'PartyRole': '2' + } + ] +} + +message_dict = { + 'OrdType': '1', + 'AccountType': '2', + 'OrderQty': '3', + 'Price': '4', + 'ClOrdID': str(cl_ord_id), + 'TransactTime': transact_time, + 'TradingParty': trading_party_dict, + 'message_type': 'NewOrderSingle', + 'direction': 'SECOND', + 'properties': {}, + 'protocol': '', + 'sequence': 0, + 'session_alias': 'jsTUURfy', + 'session_group': '', + 'subsequence': [], + 'timestamp': None +} + +# AnyMessage.raw_message + +raw_message = RawMessage(parent_event_id=parent_event_id, + metadata=RawMessageMetadata( + id=MessageID( + connection_id=ConnectionID(session_alias='jsTUURfy'), + direction=Direction.SECOND + ), + timestamp=ts, + properties={'field1': 'value1'}, + protocol='_protocol' + ), + body=os.urandom(144)) + +any_message_raw = AnyMessage(raw_message=raw_message) + +raw_message_dict = { + 'session_alias': 'jsTUURfy', + 'direction': 'SECOND', + 'protocol': '_protocol' +} + +# AnyMessage.byte_message + +byte_message = ByteMessage(parent_event_id=parent_event_id, + metadata=RawMessageMetadata( + id=MessageID( + connection_id=ConnectionID(session_alias='jsTUURfy'), + direction=Direction.SECOND + ), + timestamp=ts, + properties={'field1': 'value1'}, + protocol='_protocol' + ), + body=os.urandom(144)) + +any_message_byte = AnyMessage(byte_message=byte_message) + +byte_message_dict = { + 'session_alias': 'jsTUURfy', + 'direction': 'SECOND', + 'protocol': '_protocol' +} diff --git a/test/test_filter_strategy/resources/messages_for_filtering.py b/test/test_filter_strategy/resources/messages_for_filtering.py new file mode 100644 index 0000000..70fed20 --- /dev/null +++ b/test/test_filter_strategy/resources/messages_for_filtering.py @@ -0,0 +1,104 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from th2_grpc_common.common_pb2 import AnyMessage, ByteMessage, ConnectionID, Direction, Message, MessageGroup, \ + MessageGroupBatch, MessageID, MessageMetadata, RawMessage, RawMessageMetadata, Value + + +message_group1 = MessageGroup(messages=[ # goes to queue_AND_filter, queue_OR_filter + AnyMessage( # to QUEUE1 (AND filter) + message=Message(metadata=MessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='qwerty'))), + fields={'msg11': Value(simple_value='11')})), + AnyMessage( # to QUEUE2 (OR filter) + message=Message(fields={'msg21': Value(simple_value='21')})), + AnyMessage( # NOT to QUEUE3 (OR-AND filter) + message=Message(fields={ + 'msg31': Value(simple_value='31'), + 'msg32': Value(simple_value='thirty-one') + })) +] +) + +message_group2 = MessageGroup(messages=[ # goes to queue_OR_filter, queue_OR_AND_filter + AnyMessage( # to QUEUE2 (OR filter) + message=Message(fields={'msg21': Value(simple_value='21')})), + AnyMessage( # NOT to QUEUE1 (AND filter) + message=Message(metadata=MessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='qwerty'))), + fields={'msg11': Value(simple_value='eleven')})), + AnyMessage( # to QUEUE3 (OR-AND filter) + message=Message(fields={ + 'msg31': Value(simple_value='31'), + 'msg32': Value(simple_value='32') + })) +]) + +message_group3 = MessageGroup(messages=[ # goes to queue_AND_filter, queue_OR_filter, queue_OR_AND_filter + AnyMessage( # to QUEUE1 (AND filter) + message=Message(metadata=MessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='qwerty'))), + fields={'msg11': Value(simple_value='11')})), + AnyMessage( # to QUEUE2 (OR filter) + message=Message(fields={'msg22': Value(simple_value='22')})), + AnyMessage( # to QUEUE3 (OR-AND filter) + message=Message(metadata=MessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='asdfgh'), + direction=Direction.SECOND)))) +]) + +message_group4 = MessageGroup(messages=[ # goes to queue_OR_AND_filter + AnyMessage( # to QUEUE3 (OR-AND filter) + raw_message=RawMessage( + metadata=RawMessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='asdfgh'), + direction=Direction.SECOND)))), + AnyMessage( # NOT to QUEUE3 (OR-AND filter) + raw_message=RawMessage( + metadata=RawMessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='asdfgh'), + direction=Direction.FIRST)))) +]) + +message_group5 = MessageGroup(messages=[ # goes to queue_OR_AND_filter + AnyMessage( # to QUEUE3 (OR-AND filter) + byte_message=ByteMessage( + metadata=RawMessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='asdfgh'), + direction=Direction.SECOND)))), + AnyMessage( # NOT to QUEUE3 (OR-AND filter) + byte_message=ByteMessage( + metadata=RawMessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias='asdfgh'), + direction=Direction.FIRST)))) +]) + +message_group6 = MessageGroup(messages=[ # goes to queue_EMPTY_filter + AnyMessage( # to QUEUE4 (EMPTY filter) + raw_message=RawMessage()), + AnyMessage( # to QUEUE4 (EMPTY filter) + message=Message()) +]) + +message_group_batch = MessageGroupBatch(groups=[ + message_group1, + message_group2, + message_group3, + message_group4, + message_group5, + message_group6 +]) + +filtered_by_queue = { + 'queue_AND_filter': MessageGroupBatch( + groups=[message_group1, message_group3]), + 'queue_OR_filter': MessageGroupBatch( + groups=[message_group1, message_group2, message_group3]), + 'queue_OR_AND_filter': MessageGroupBatch( + groups=[message_group2, message_group3, message_group4, message_group5]), + 'queue_EMPTY_filter': MessageGroupBatch( + groups=[message_group1, message_group2, message_group3, message_group4, message_group5, message_group6]) +} diff --git a/test/test_filter_strategy/test_default_filter_strategy.py b/test/test_filter_strategy/test_default_filter_strategy.py new file mode 100644 index 0000000..2d9cf99 --- /dev/null +++ b/test/test_filter_strategy/test_default_filter_strategy.py @@ -0,0 +1,61 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from test.test_filter_strategy.resources.messages_for_extraction import any_message, any_message_byte, \ + any_message_raw, byte_message_dict, message_dict, raw_message_dict +from test.test_filter_strategy.resources.messages_for_filtering import filtered_by_queue +from test.utils import object_to_dict +from typing import Any, Dict + +import pytest +from th2_common.schema.strategy.field_extraction.th2_msg_field_extraction import Th2MsgFieldExtraction + + +def test_message_field_extraction() -> None: + extract_strategy = Th2MsgFieldExtraction() + assert extract_strategy.get_fields(any_message) == message_dict + + +def test_raw_message_field_extraction() -> None: + extract_strategy = Th2MsgFieldExtraction() + assert extract_strategy.get_fields(any_message_raw) == raw_message_dict + + +def test_byte_message_field_extraction() -> None: + extract_strategy = Th2MsgFieldExtraction() + assert extract_strategy.get_fields(any_message_byte) == byte_message_dict + + +@pytest.mark.usefixtures('filtered_messages') +def test_default_filter_strategy_and_filter(filtered_messages: Dict[str, Any]) -> None: + queue = 'queue_AND_filter' + assert object_to_dict(filtered_messages)[queue] == object_to_dict(filtered_by_queue)[queue] + + +@pytest.mark.usefixtures('filtered_messages') +def test_default_filter_strategy_or_filter(filtered_messages: Dict[str, Any]) -> None: + queue = 'queue_OR_filter' + assert object_to_dict(filtered_messages)[queue] == object_to_dict(filtered_by_queue)[queue] + + +@pytest.mark.usefixtures('filtered_messages') +def test_default_filter_strategy_or_and_filter(filtered_messages: Dict[str, Any]) -> None: + queue = 'queue_OR_AND_filter' + assert object_to_dict(filtered_messages)[queue] == object_to_dict(filtered_by_queue)[queue] + + +@pytest.mark.usefixtures('filtered_messages') +def test_default_filter_strategy_empty_filter(filtered_messages: Dict[str, Any]) -> None: + queue = 'queue_EMPTY_filter' + assert object_to_dict(filtered_messages)[queue] == object_to_dict(filtered_by_queue)[queue] diff --git a/test/test_filter_strategy/test_default_grpc_filter_strategy.py b/test/test_filter_strategy/test_default_grpc_filter_strategy.py new file mode 100644 index 0000000..6e7d6a7 --- /dev/null +++ b/test/test_filter_strategy/test_default_grpc_filter_strategy.py @@ -0,0 +1,63 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock + +import pytest +from th2_common.schema.exception.grpc_router_error import GrpcRouterError +from th2_common.schema.grpc.router.impl.default_grpc_router import DefaultGrpcRouter + + +@pytest.mark.usefixtures('grpc_router') +def test_default_grpc_filter_strategy_simple_filter(grpc_router: DefaultGrpcRouter) -> None: + services = grpc_router._filter_services_by_name(service_class_name='ServiceClass1') + connection = DefaultGrpcRouter.Connection(services=services, stub_class=Mock(), channels=Mock(), options=Mock()) + + filtered_service = connection._filter_services(properties={'session_alias': 'asdfgh', 'msg11': ''}) + + assert filtered_service.service_class == 'ServiceClass1' + + +@pytest.mark.usefixtures('grpc_router') +def test_default_grpc_filter_strategy_multiple_filters(grpc_router: DefaultGrpcRouter) -> None: + services = grpc_router._filter_services_by_name(service_class_name='ServiceClass2') + connection = DefaultGrpcRouter.Connection(services=services, stub_class=Mock(), channels=Mock(), options=Mock()) + + filtered_service = connection._filter_services(properties={'prop21': '21', 'prop22': '22', 'prop23': '23'}) + + assert filtered_service.service_class == 'ServiceClass2' + + +@pytest.mark.usefixtures('grpc_router') +def test_default_grpc_filter_strategy_multiple_services(grpc_router: DefaultGrpcRouter) -> None: + services = grpc_router._filter_services_by_name(service_class_name='ServiceClass3') + connection = DefaultGrpcRouter.Connection(services=services, stub_class=Mock(), channels=Mock(), options=Mock()) + + try: + connection._filter_services(properties={'prop31': '31', 'prop32': '32'}) + raise AssertionError() + except GrpcRouterError: + assert True # two services pass the filter -> GrpcRouterError + + +@pytest.mark.usefixtures('grpc_router') +def test_default_grpc_filter_strategy_no_services(grpc_router: DefaultGrpcRouter) -> None: + services = grpc_router._filter_services_by_name(service_class_name='ServiceClass3') + connection = DefaultGrpcRouter.Connection(services=services, stub_class=Mock(), channels=Mock(), options=Mock()) + + try: + connection._filter_services(properties={'any_filed': '0'}) + raise AssertionError() + except GrpcRouterError: + assert True # no services pass the filter -> GrpcRouterError diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 0000000..9c8692b --- /dev/null +++ b/test/utils.py @@ -0,0 +1,29 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Any, Dict, ItemsView + + +def object_to_dict(obj: Any) -> Dict[str, Any]: + if isinstance(obj, Enum): + return obj.value # type: ignore + elif hasattr(obj, '__dict__'): + return {k: object_to_dict(v) for k, v in obj.__dict__.items()} + elif isinstance(obj, dict): + return {k: object_to_dict(v) for k, v in obj.items()} + elif isinstance(obj, (list, ItemsView)): + return list(map(object_to_dict, obj)) # type: ignore + else: + return obj # type: ignore diff --git a/th2_common/__init__.py b/th2_common/__init__.py index d14c5ce..5fee55d 100644 --- a/th2_common/__init__.py +++ b/th2_common/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pkg_resources import get_distribution, DistributionNotFound - +from pkg_resources import DistributionNotFound, get_distribution from th2_common_utils.message_fields_access import * try: print(f"Using th2-common=={get_distribution('th2_common').version}") except DistributionNotFound: - print(f'th2-common package not installed') + print('th2-common package is not installed') diff --git a/th2_common/schema/box/configuration/box_configuration.py b/th2_common/schema/box/configuration/box_configuration.py index 3977877..f0d49e3 100644 --- a/th2_common/schema/box/configuration/box_configuration.py +++ b/th2_common/schema/box/configuration/box_configuration.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any + from th2_common.schema.configuration.abstract_configuration import AbstractConfiguration class BoxConfiguration(AbstractConfiguration): - def __init__(self, box_name, **kwargs): + + def __init__(self, box_name: str, **kwargs: Any) -> None: self.boxName = box_name self.check_unexpected_args(kwargs) diff --git a/th2_common/schema/configuration/abstract_configuration.py b/th2_common/schema/configuration/abstract_configuration.py index a37170a..515723e 100644 --- a/th2_common/schema/configuration/abstract_configuration.py +++ b/th2_common/schema/configuration/abstract_configuration.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from abc import ABC - +import logging +from typing import Any, Dict logger = logging.getLogger(__name__) class AbstractConfiguration(ABC): - def check_unexpected_args(self, args): - if len(args) > 0: - logger.warning(f'{self.__class__.__name__} JSON config contains unexpected arguments: {args}') + + def check_unexpected_args(self, kwargs: Dict[str, Any]) -> None: + if len(kwargs) > 0: + logger.warning(f'{self.__class__.__name__} JSON config contains unexpected arguments: {kwargs}') diff --git a/th2_common/schema/cradle/cradle_configuration.py b/th2_common/schema/cradle/cradle_configuration.py index 3901f2b..a7d6c00 100644 --- a/th2_common/schema/cradle/cradle_configuration.py +++ b/th2_common/schema/cradle/cradle_configuration.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,13 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any + from th2_common.schema.configuration.abstract_configuration import AbstractConfiguration class CradleConfiguration(AbstractConfiguration): - def __init__(self, dataCenter: str, host: str, port: int, - keyspace: str, username: str, password: str, **kwargs) -> None: + def __init__(self, + dataCenter: str, + host: str, + port: int, + keyspace: str, + username: str, + password: str, + **kwargs: Any) -> None: self.data_center = dataCenter self.host = host self.port = port diff --git a/th2_common/schema/event/event_batch_router.py b/th2_common/schema/event/event_batch_router.py index 9f38d41..68c9b93 100644 --- a/th2_common/schema/event/event_batch_router.py +++ b/th2_common/schema/event/event_batch_router.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict -from th2_grpc_common.common_pb2 import EventBatch +from collections import defaultdict +from typing import Dict, Set +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer from th2_common.schema.event.event_batch_sender import EventBatchSender from th2_common.schema.event.event_batch_subscriber import EventBatchSubscriber from th2_common.schema.message.configuration.message_configuration import QueueConfiguration @@ -24,46 +25,55 @@ from th2_common.schema.message.message_sender import MessageSender from th2_common.schema.message.message_subscriber import MessageSubscriber from th2_common.schema.message.queue_attribute import QueueAttribute +from th2_grpc_common.common_pb2 import Event, EventBatch, Message class EventBatchRouter(AbstractRabbitMessageRouter): - def _get_messages(self, batch) -> list: + def _get_messages(self, batch: EventBatch) -> RepeatedCompositeFieldContainer: return batch.events - def _create_batch(self): + def _create_batch(self) -> EventBatch: return EventBatch() - def _add_message(self, batch, message): + def _add_message(self, batch: EventBatch, message: Event) -> None: batch.events.append(message) - def update_dropped_metrics(self, batch, pin): + def update_dropped_metrics(self, batch: EventBatch, *pins: str) -> None: pass - def split_and_filter(self, queue_aliases_to_configs, batch) -> dict: - result = defaultdict(EventBatch) + def split_and_filter(self, + queue_aliases_to_configs: Dict[str, QueueConfiguration], + batch: EventBatch) -> Dict[str, EventBatch]: + result: Dict[str, EventBatch] = defaultdict(EventBatch) for message in self._get_messages(batch): - for queue_alias in queue_aliases_to_configs.keys(): + for queue_alias in queue_aliases_to_configs: self._add_message(result[queue_alias], message) return result @property - def required_subscribe_attributes(self): - return {QueueAttribute.SUBSCRIBE.value, QueueAttribute.EVENT.value} + def required_subscribe_attributes(self) -> Set[str]: + return {QueueAttribute.SUBSCRIBE, QueueAttribute.EVENT} @property - def required_send_attributes(self): - return {QueueAttribute.PUBLISH.value, QueueAttribute.EVENT.value} + def required_send_attributes(self) -> Set[str]: + return {QueueAttribute.PUBLISH, QueueAttribute.EVENT} - def _find_by_filter(self, queues: {str: QueueConfiguration}, msg) -> dict: - return {key: msg for key in queues.keys()} + def _find_by_filter(self, queues: Dict[str, QueueConfiguration], msg: Message) -> Dict[str, Message]: + return {key: msg for key in queues} - def create_sender(self, connection_manager: ConnectionManager, - queue_configuration: QueueConfiguration, th2_pin) -> MessageSender: - return EventBatchSender(connection_manager, queue_configuration.exchange, queue_configuration.routing_key, + def create_sender(self, + connection_manager: ConnectionManager, + queue_configuration: QueueConfiguration, + th2_pin: str) -> MessageSender: + return EventBatchSender(connection_manager, + queue_configuration.exchange, + queue_configuration.routing_key, th2_pin=th2_pin) - def create_subscriber(self, connection_manager: ConnectionManager, - queue_configuration: QueueConfiguration, th2_pin) -> MessageSubscriber: + def create_subscriber(self, + connection_manager: ConnectionManager, + queue_configuration: QueueConfiguration, + th2_pin: str) -> MessageSubscriber: subscribe_target = SubscribeTarget(queue_configuration.queue, queue_configuration.routing_key) - return EventBatchSubscriber(connection_manager, queue_configuration, subscribe_target, th2_pin=th2_pin) + return EventBatchSubscriber(connection_manager, subscribe_target, queue_configuration, th2_pin=th2_pin) diff --git a/th2_common/schema/event/event_batch_sender.py b/th2_common/schema/event/event_batch_sender.py index bf7a39f..86866d7 100644 --- a/th2_common/schema/event/event_batch_sender.py +++ b/th2_common/schema/event/event_batch_sender.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer from google.protobuf.json_format import MessageToJson from prometheus_client import Counter -from th2_grpc_common.common_pb2 import EventBatch - from th2_common.schema.message.impl.rabbitmq.abstract_rabbit_sender import AbstractRabbitSender import th2_common.schema.metrics.common_metrics as common_metrics -from th2_common.schema.metrics.metric_utils import update_total_metrics from th2_common.schema.util.util import get_debug_string_event +from th2_grpc_common.common_pb2 import EventBatch class EventBatchSender(AbstractRabbitSender): @@ -29,19 +28,19 @@ class EventBatchSender(AbstractRabbitSender): _TH2_TYPE = 'EVENT' - def send(self, message): + def send(self, message: EventBatch) -> None: self.OUTGOING_EVENT_QUANTITY.labels(self.th2_pin).inc(len(message.events)) super().send(message) - def get_events(self, batch: EventBatch) -> list: + def get_events(self, batch: EventBatch) -> RepeatedCompositeFieldContainer: return batch.events @staticmethod - def value_to_bytes(value: EventBatch): + def value_to_bytes(value: EventBatch) -> bytes: return value.SerializeToString() - def to_trace_string(self, value): + def to_trace_string(self, value: EventBatch) -> str: return MessageToJson(value) - def to_debug_string(self, value): + def to_debug_string(self, value: EventBatch) -> str: return get_debug_string_event(value) diff --git a/th2_common/schema/event/event_batch_subscriber.py b/th2_common/schema/event/event_batch_subscriber.py index 880c4c9..57a4bca 100644 --- a/th2_common/schema/event/event_batch_subscriber.py +++ b/th2_common/schema/event/event_batch_subscriber.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer from google.protobuf.json_format import MessageToJson -from prometheus_client import Counter, Histogram -from th2_grpc_common.common_pb2 import EventBatch - -import th2_common.schema.metrics.common_metrics as common_metrics +from prometheus_client import Counter from th2_common.schema.message.impl.rabbitmq.abstract_rabbit_subscriber import AbstractRabbitSubscriber +import th2_common.schema.metrics.common_metrics as common_metrics from th2_common.schema.util.util import get_debug_string_event +from th2_grpc_common.common_pb2 import EventBatch class EventBatchSubscriber(AbstractRabbitSubscriber): @@ -29,26 +29,26 @@ class EventBatchSubscriber(AbstractRabbitSubscriber): _th2_type = 'EVENT' - def get_events(self, batch: EventBatch) -> list: + def get_events(self, batch: EventBatch) -> RepeatedCompositeFieldContainer: return batch.events @staticmethod - def value_from_bytes(body): + def value_from_bytes(body: bytes) -> EventBatch: event_batch = EventBatch() event_batch.ParseFromString(body) return event_batch - def filter(self, value) -> bool: + def filter(self, value: EventBatch) -> bool: # noqa: A003 return True - def to_trace_string(self, value): + def to_trace_string(self, value: EventBatch) -> str: return MessageToJson(value) - def to_debug_string(self, value): + def to_debug_string(self, value: EventBatch) -> str: return get_debug_string_event(value) - def update_dropped_metrics(self, batch): + def update_dropped_metrics(self, batch: EventBatch) -> None: pass - def update_total_metrics(self, batch): + def update_total_metrics(self, batch: EventBatch) -> None: self.INCOMING_EVENTS_QUANTITY.labels(self.th2_pin).inc(len(batch.events)) diff --git a/th2_common/schema/factory/abstract_common_factory.py b/th2_common/schema/factory/abstract_common_factory.py index dba58a4..93356cc 100644 --- a/th2_common/schema/factory/abstract_common_factory.py +++ b/th2_common/schema/factory/abstract_common_factory.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,22 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +from abc import ABC, abstractmethod import json import logging.config import os -from abc import ABC, abstractmethod from pathlib import Path -from threading import Lock +from typing import Any, Dict, List, Optional, Type, Union -import th2_common.schema.metrics.common_metrics as common_metrics from th2_common.schema.cradle.cradle_configuration import CradleConfiguration from th2_common.schema.event.event_batch_router import EventBatchRouter -from th2_common.schema.grpc.configuration.grpc_configuration import GrpcConfiguration, GrpcRouterConfiguration +from th2_common.schema.grpc.configuration.grpc_configuration import GrpcConfiguration, GrpcConnectionConfiguration from th2_common.schema.grpc.router.grpc_router import GrpcRouter from th2_common.schema.grpc.router.impl.default_grpc_router import DefaultGrpcRouter from th2_common.schema.log.trace import install_trace_logger from th2_common.schema.message.configuration.message_configuration import MessageRouterConfiguration, \ - ConnectionManagerConfiguration + MqConnectionConfiguration +from th2_common.schema.message.impl.rabbitmq.byte.rabbit_cbor_batch_router import RabbitCborBatchRouter +from th2_common.schema.message.impl.rabbitmq.byte.rabbit_json_batch_router import RabbitJsonBatchRouter from th2_common.schema.message.impl.rabbitmq.configuration.rabbitmq_configuration import RabbitMQConfiguration from th2_common.schema.message.impl.rabbitmq.connection.connection_manager import ConnectionManager from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router import \ @@ -35,6 +36,7 @@ from th2_common.schema.message.impl.rabbitmq.parsed.rabbit_parsed_batch_router import RabbitParsedBatchRouter from th2_common.schema.message.impl.rabbitmq.raw.rabbit_raw_batch_router import RabbitRawBatchRouter from th2_common.schema.message.message_router import MessageRouter +import th2_common.schema.metrics.common_metrics as common_metrics from th2_common.schema.metrics.prometheus_configuration import PrometheusConfiguration from th2_common.schema.metrics.prometheus_server import PrometheusServer @@ -46,35 +48,41 @@ class AbstractCommonFactory(ABC): LOGGING_CONFIG_FILENAME = 'log4py.conf' DEFAULT_LOGGING_CONFIG_OUTER_PATH = Path('/var/th2/config/') / LOGGING_CONFIG_FILENAME - DEFAULT_LOGGING_CONFIG_INNER_PATH = Path(__file__).parent.parent / 'log' / LOGGING_CONFIG_FILENAME + DEFAULT_LOGGING_CONFIG_INNER_PATH = Path(__file__).parent.parent / 'log' / 'log_config.json' def __init__(self, - message_parsed_batch_router_class=RabbitParsedBatchRouter, - message_raw_batch_router_class=RabbitRawBatchRouter, - message_group_batch_router_class=RabbitMessageGroupBatchRouter, - event_batch_router_class=EventBatchRouter, - grpc_router_class=DefaultGrpcRouter, - logging_config_filepath=None) -> None: - - self.rabbit_mq_configuration = None - self.message_router_configuration = None - self.grpc_configuration = None - self.grpc_router_configuration = None - - self._connection_manager = None - self.connection_manager_configuration = None + message_parsed_batch_router_class: Type[RabbitParsedBatchRouter] = RabbitParsedBatchRouter, + message_raw_batch_router_class: Type[RabbitRawBatchRouter] = RabbitRawBatchRouter, + message_group_batch_router_class: Type[RabbitMessageGroupBatchRouter] = RabbitMessageGroupBatchRouter, + message_cbor_router_class: Type[RabbitCborBatchRouter] = RabbitCborBatchRouter, + message_json_router_class: Type[RabbitJsonBatchRouter] = RabbitJsonBatchRouter, + event_batch_router_class: Type[EventBatchRouter] = EventBatchRouter, + grpc_router_class: Type[DefaultGrpcRouter] = DefaultGrpcRouter, + logging_config_filepath: Optional[Path] = None) -> None: + + self.rabbit_mq_configuration: Optional[RabbitMQConfiguration] = None + self.message_router_configuration: Optional[MessageRouterConfiguration] = None + self.grpc_configuration: Optional[GrpcConfiguration] = None + self.grpc_connection_configuration: Optional[GrpcConnectionConfiguration] = None + + self._connection_manager: Optional[ConnectionManager] = None + self.connection_manager_configuration: Optional[MqConnectionConfiguration] = None self.message_parsed_batch_router_class = message_parsed_batch_router_class self.message_raw_batch_router_class = message_raw_batch_router_class self.message_group_batch_router_class = message_group_batch_router_class + self.message_cbor_router_class = message_cbor_router_class + self.message_json_router_class = message_json_router_class self.event_batch_router_class = event_batch_router_class self.grpc_router_class = grpc_router_class - self._message_parsed_batch_router = None - self._message_raw_batch_router = None - self._message_group_batch_router = None - self._event_batch_router = None - self._grpc_router = None + self._message_parsed_batch_router: Optional[MessageRouter] = None + self._message_raw_batch_router: Optional[MessageRouter] = None + self._message_group_batch_router: Optional[MessageRouter] = None + self._message_cbor_router: Optional[MessageRouter] = None + self._message_json_router: Optional[MessageRouter] = None + self._event_batch_router: Optional[MessageRouter] = None + self._grpc_router: Optional[GrpcRouter] = None install_trace_logger() @@ -87,8 +95,9 @@ def __init__(self, disable_existing_loggers=False) logger.info(f'Using logging config file from {AbstractCommonFactory.DEFAULT_LOGGING_CONFIG_OUTER_PATH}') elif AbstractCommonFactory.DEFAULT_LOGGING_CONFIG_INNER_PATH.exists(): - logging.config.fileConfig(fname=AbstractCommonFactory.DEFAULT_LOGGING_CONFIG_INNER_PATH, - disable_existing_loggers=False) + logging.config.dictConfig( + self.read_configuration(Path(AbstractCommonFactory.DEFAULT_LOGGING_CONFIG_INNER_PATH)) + ) logger.info(f'Using logging config file from {AbstractCommonFactory.DEFAULT_LOGGING_CONFIG_INNER_PATH}') self._liveness_monitor = common_metrics.register_liveness('common_factory_liveness') @@ -159,6 +168,46 @@ def message_group_batch_router(self) -> MessageRouter: return self._message_group_batch_router + @property + def message_cbor_router(self) -> MessageRouter: + """ + Create MessageRouter which work with dicts + """ + if self._connection_manager is None: + if self.rabbit_mq_configuration is None: + self.rabbit_mq_configuration = self._create_rabbit_mq_configuration() + if self.connection_manager_configuration is None: + self.connection_manager_configuration = self._create_conn_manager_configuration() + self._connection_manager = ConnectionManager(self.rabbit_mq_configuration, + self.connection_manager_configuration) + if self.message_router_configuration is None: + self.message_router_configuration = self._create_message_router_configuration() + if self._message_cbor_router is None: + self._message_cbor_router = self.message_cbor_router_class(self._connection_manager, + self.message_router_configuration) + + return self._message_cbor_router + + @property + def message_json_router(self) -> MessageRouter: + """ + Create MessageRouter which work with dicts + """ + if self._connection_manager is None: + if self.rabbit_mq_configuration is None: + self.rabbit_mq_configuration = self._create_rabbit_mq_configuration() + if self.connection_manager_configuration is None: + self.connection_manager_configuration = self._create_conn_manager_configuration() + self._connection_manager = ConnectionManager(self.rabbit_mq_configuration, + self.connection_manager_configuration) + if self.message_router_configuration is None: + self.message_router_configuration = self._create_message_router_configuration() + if self._message_json_router is None: + self._message_json_router = self.message_json_router_class(self._connection_manager, + self.message_router_configuration) + + return self._message_json_router + @property def event_batch_router(self) -> MessageRouter: """ @@ -184,13 +233,13 @@ def grpc_router(self) -> GrpcRouter: if self._grpc_router is None: if self.grpc_configuration is None: self.grpc_configuration = self._create_grpc_configuration() - if self.grpc_router_configuration is None: - self.grpc_router_configuration = self._create_grpc_router_configuration() - self._grpc_router = self.grpc_router_class(self.grpc_configuration, self.grpc_router_configuration) + if self.grpc_connection_configuration is None: + self.grpc_connection_configuration = self._create_grpc_router_configuration() + self._grpc_router = self.grpc_router_class(self.grpc_configuration, self.grpc_connection_configuration) return self._grpc_router - def close(self): + def close(self) -> None: logger.info('Closing Common Factory') if self._message_raw_batch_router is not None: @@ -234,13 +283,13 @@ def close(self): logger.info('Common Factory is closed') @staticmethod - def read_configuration(filepath): + def read_configuration(filepath: Path) -> Dict[str, Any]: with open(filepath, 'r') as file: config_json = file.read() config_json_expanded = os.path.expandvars(config_json) - config_dict = json.loads(config_json_expanded) + json_object = json.loads(config_json_expanded) - return config_dict + return json_replace_keys(json_object) # type: ignore def create_cradle_configuration(self) -> CradleConfiguration: return CradleConfiguration(**self.read_configuration(self._path_to_cradle_configuration)) @@ -251,7 +300,7 @@ def _create_prometheus_configuration(self) -> PrometheusConfiguration: else: return PrometheusConfiguration() - def create_custom_configuration(self) -> dict: + def create_custom_configuration(self) -> Any: return self.read_configuration(self._path_to_custom_configuration) def _create_rabbit_mq_configuration(self) -> RabbitMQConfiguration: @@ -264,23 +313,26 @@ def _create_message_router_configuration(self) -> MessageRouterConfiguration: self.message_router_configuration = MessageRouterConfiguration(**config_dict) return self.message_router_configuration - def _create_conn_manager_configuration(self) -> ConnectionManagerConfiguration: + def _create_conn_manager_configuration(self) -> MqConnectionConfiguration: if self._path_to_connection_manager_configuration.exists(): - return ConnectionManagerConfiguration(**self.read_configuration( - self._path_to_connection_manager_configuration)) + return MqConnectionConfiguration( + **self.read_configuration(self._path_to_connection_manager_configuration) + ) else: - return ConnectionManagerConfiguration() + return MqConnectionConfiguration() def _create_grpc_configuration(self) -> GrpcConfiguration: config_dict = self.read_configuration(self._path_to_grpc_configuration) self.grpc_configuration = GrpcConfiguration(**config_dict) return self.grpc_configuration - def _create_grpc_router_configuration(self) -> GrpcRouterConfiguration: + def _create_grpc_router_configuration(self) -> GrpcConnectionConfiguration: if self._path_to_grpc_router_configuration.exists(): - return GrpcRouterConfiguration(**self.read_configuration(self._path_to_grpc_router_configuration)) + return GrpcConnectionConfiguration( + **self.read_configuration(self._path_to_grpc_router_configuration) + ) else: - return GrpcRouterConfiguration() + return GrpcConnectionConfiguration() @property @abstractmethod @@ -321,3 +373,12 @@ def _path_to_prometheus_configuration(self) -> Path: @abstractmethod def _path_to_custom_configuration(self) -> Path: pass + + +def json_replace_keys(json_object: Union[str, int, float, List, Dict]) -> Union[str, int, float, List, Dict[str, Any]]: + if isinstance(json_object, Dict): + return {k.replace('-', '_'): json_replace_keys(v) for k, v in json_object.items()} + elif isinstance(json_object, List): + return [json_replace_keys(i) for i in json_object] + else: + return json_object diff --git a/th2_common/schema/factory/common_factory.py b/th2_common/schema/factory/common_factory.py index 6fabf5b..e85f185 100644 --- a/th2_common/schema/factory/common_factory.py +++ b/th2_common/schema/factory/common_factory.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,15 +17,18 @@ import json import logging import os -import sys -from os import mkdir, getcwd +from os import getcwd, mkdir from pathlib import Path +import sys +from typing import Any, Dict, List, Optional, Type from kubernetes import client, config - +from kubernetes.client import V1ConfigMapList from th2_common.schema.event.event_batch_router import EventBatchRouter from th2_common.schema.factory.abstract_common_factory import AbstractCommonFactory from th2_common.schema.grpc.router.impl.default_grpc_router import DefaultGrpcRouter +from th2_common.schema.message.impl.rabbitmq.byte.rabbit_cbor_batch_router import RabbitCborBatchRouter +from th2_common.schema.message.impl.rabbitmq.byte.rabbit_json_batch_router import RabbitJsonBatchRouter from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router import \ RabbitMessageGroupBatchRouter from th2_common.schema.message.impl.rabbitmq.parsed.rabbit_parsed_batch_router import RabbitParsedBatchRouter @@ -58,29 +61,30 @@ class CommonFactory(AbstractCommonFactory): # FIX: Add path to dictionary as a parameter def __init__(self, - config_path=None, - rabbit_mq_config_filepath=CONFIG_DEFAULT_PATH / RABBIT_MQ_CONFIG_FILENAME, - mq_router_config_filepath=CONFIG_DEFAULT_PATH / MQ_ROUTER_CONFIG_FILENAME, - connection_manager_config_filepath=CONFIG_DEFAULT_PATH / CONNECTION_MANAGER_CONFIG_FILENAME, - grpc_config_filepath=CONFIG_DEFAULT_PATH / GRPC_CONFIG_FILENAME, - grpc_router_config_filepath=CONFIG_DEFAULT_PATH / GRPC_ROUTER_CONFIG_FILENAME, - cradle_config_filepath=CONFIG_DEFAULT_PATH / CRADLE_CONFIG_FILENAME, - prometheus_config_filepath=CONFIG_DEFAULT_PATH / PROMETHEUS_CONFIG_FILENAME, - custom_config_filepath=CONFIG_DEFAULT_PATH / CUSTOM_CONFIG_FILENAME, - logging_config_filepath=None, - - message_parsed_batch_router_class=RabbitParsedBatchRouter, - message_raw_batch_router_class=RabbitRawBatchRouter, - message_group_batch_router_class=RabbitMessageGroupBatchRouter, - event_batch_router_class=EventBatchRouter, - grpc_router_class=DefaultGrpcRouter) -> None: + config_path: Optional[Path] = None, + rabbit_mq_config_filepath: Path = CONFIG_DEFAULT_PATH / RABBIT_MQ_CONFIG_FILENAME, + mq_router_config_filepath: Path = CONFIG_DEFAULT_PATH / MQ_ROUTER_CONFIG_FILENAME, + connection_manager_config_filepath: Path = CONFIG_DEFAULT_PATH / CONNECTION_MANAGER_CONFIG_FILENAME, + grpc_config_filepath: Path = CONFIG_DEFAULT_PATH / GRPC_CONFIG_FILENAME, + grpc_router_config_filepath: Path = CONFIG_DEFAULT_PATH / GRPC_ROUTER_CONFIG_FILENAME, + cradle_config_filepath: Path = CONFIG_DEFAULT_PATH / CRADLE_CONFIG_FILENAME, + prometheus_config_filepath: Path = CONFIG_DEFAULT_PATH / PROMETHEUS_CONFIG_FILENAME, + custom_config_filepath: Path = CONFIG_DEFAULT_PATH / CUSTOM_CONFIG_FILENAME, + logging_config_filepath: Optional[Path] = None, + + message_parsed_batch_router_class: Type[RabbitParsedBatchRouter] = RabbitParsedBatchRouter, + message_raw_batch_router_class: Type[RabbitRawBatchRouter] = RabbitRawBatchRouter, + message_group_batch_router_class: Type[RabbitMessageGroupBatchRouter] = RabbitMessageGroupBatchRouter, + message_cbor_router_class: Type[RabbitCborBatchRouter] = RabbitCborBatchRouter, + message_json_router_class: Type[RabbitJsonBatchRouter] = RabbitJsonBatchRouter, + event_batch_router_class: Type[EventBatchRouter] = EventBatchRouter, + grpc_router_class: Type[DefaultGrpcRouter] = DefaultGrpcRouter) -> None: if config_path is not None: config_path = Path(config_path) rabbit_mq_config_filepath = config_path / CommonFactory.RABBIT_MQ_CONFIG_FILENAME mq_router_config_filepath = config_path / CommonFactory.MQ_ROUTER_CONFIG_FILENAME - connection_manager_config_filepath = config_path / \ - CommonFactory.CONNECTION_MANAGER_CONFIG_FILENAME + connection_manager_config_filepath = config_path / CommonFactory.CONNECTION_MANAGER_CONFIG_FILENAME grpc_config_filepath = config_path / CommonFactory.GRPC_CONFIG_FILENAME grpc_router_config_filepath = config_path / CommonFactory.GRPC_ROUTER_CONFIG_FILENAME cradle_config_filepath = config_path / CommonFactory.CRADLE_CONFIG_FILENAME @@ -98,15 +102,16 @@ def __init__(self, self.custom_config_filepath = Path(custom_config_filepath) super().__init__(message_parsed_batch_router_class, message_raw_batch_router_class, - message_group_batch_router_class, event_batch_router_class, grpc_router_class, + message_group_batch_router_class, message_cbor_router_class, message_json_router_class, + event_batch_router_class, grpc_router_class, logging_config_filepath) @staticmethod - def calculate_path(parsed_args, name_attr, path_default) -> Path: + def calculate_path(parsed_args: argparse.Namespace, name_attr: str, path_default: str) -> Path: return getattr(parsed_args, name_attr, CommonFactory.CONFIG_DEFAULT_PATH / path_default) @staticmethod - def create_from_arguments(args=None): + def create_from_arguments(args: Optional[List[str]] = None) -> 'CommonFactory': if args is None: args = sys.argv[1:] @@ -148,9 +153,11 @@ def create_from_arguments(args=None): return CommonFactory(config_path=result.configPath) else: return CommonFactory( - rabbit_mq_config_filepath=CommonFactory.calculate_path(result, 'rabbitConfiguration', + rabbit_mq_config_filepath=CommonFactory.calculate_path(result, + 'rabbitConfiguration', CommonFactory.RABBIT_MQ_CONFIG_FILENAME), - mq_router_config_filepath=CommonFactory.calculate_path(result, 'messageRouterConfiguration', + mq_router_config_filepath=CommonFactory.calculate_path(result, + 'messageRouterConfiguration', CommonFactory.MQ_ROUTER_CONFIG_FILENAME), connection_manager_config_filepath=CommonFactory.calculate_path( result, @@ -170,7 +177,7 @@ def create_from_arguments(args=None): ) @staticmethod - def create_from_kubernetes(namespace, box_name, context_name=None): + def create_from_kubernetes(namespace: str, box_name: str, context_name: Any = None) -> 'CommonFactory': config.load_kube_config(context=context_name) @@ -192,11 +199,11 @@ def create_from_kubernetes(namespace, box_name, context_name=None): prometheus_path = config_dir / CommonFactory.PROMETHEUS_CONFIG_FILENAME box_configuration_path = config_dir / CommonFactory.BOX_FILENAME - rabbit_mq_encoded_password = v1.read_namespaced_secret(CommonFactory.RABBITMQ_SECRET_NAME, namespace).data \ - .get(CommonFactory.RABBITMQ_PASSWORD_KEY) + rabbit_mq_encoded_password = v1.read_namespaced_secret( + CommonFactory.RABBITMQ_SECRET_NAME, namespace).data.get(CommonFactory.RABBITMQ_PASSWORD_KEY) - cassandra_encoded_password = v1.read_namespaced_secret(CommonFactory.CASSANDRA_SECRET_NAME, namespace).data \ - .get(CommonFactory.CASSANDRA_PASSWORD_KEY) + cassandra_encoded_password = v1.read_namespaced_secret( + CommonFactory.CASSANDRA_SECRET_NAME, namespace).data.get(CommonFactory.CASSANDRA_PASSWORD_KEY) os.environ[CommonFactory.KEY_RABBITMQ_PASS] = CommonFactory._decode_from_base64(rabbit_mq_encoded_password) os.environ[CommonFactory.KEY_CASSANDRA_PASS] = CommonFactory._decode_from_base64(cassandra_encoded_password) @@ -233,8 +240,10 @@ def create_from_kubernetes(namespace, box_name, context_name=None): CommonFactory._get_dictionary(box_name, v1.list_config_map_for_all_namespaces(), dictionary_path) - CommonFactory._get_box_config(config_maps_dict, f'{box_name}-app-config', - CommonFactory.BOX_FILENAME, box_configuration_path) + CommonFactory._get_box_config(config_maps_dict, + f'{box_name}-app-config', + CommonFactory.BOX_FILENAME, + box_configuration_path) return CommonFactory( rabbit_mq_config_filepath=rabbit_path, @@ -248,27 +257,30 @@ def create_from_kubernetes(namespace, box_name, context_name=None): ) @staticmethod - def _decode_from_base64(data): + def _decode_from_base64(data: str) -> str: data_bytes = data.encode('ascii') data_string_bytes = base64.b64decode(data_bytes) return data_string_bytes.decode('ascii') @staticmethod - def _get_dictionary(box_name, config_maps, dictionary_path): + def _get_dictionary(box_name: str, config_maps: V1ConfigMapList, dictionary_path: Path) -> None: if 'items' in config_maps.to_dict()['items']: try: for config_map in config_maps.to_dict()['items']: - if config_map['metadata']['name'].startswith(box_name) & \ + if config_map['metadata']['name'].startswith(box_name) and \ config_map['metadata']['name'].endswith('-dictionary'): with open(dictionary_path, 'w') as dictionary_file: json.dump(config_map, dictionary_file) except KeyError: - logger.error(f'dictionary config map\'s metadata not valid. Some keys are absent.') + logger.error("Dictionary config map's metadata is not valid. Some keys are absent.") except IOError: logger.error('Failed to write file for dictionary.') @staticmethod - def _get_config(config_maps_dict, name, config_file_name, path): + def _get_config(config_maps_dict: Dict[str, Dict], + name: str, + config_file_name: str, + path: Path) -> None: try: if 'items' in config_maps_dict: for config_map in config_maps_dict['items']: @@ -279,12 +291,15 @@ def _get_config(config_maps_dict, name, config_file_name, path): with open(path, 'w') as file: json.dump(config_data, file) except KeyError: - logger.error(f'{name}\'s data not valid. Some keys are absent.') + logger.error(f"{name}'s data not valid. Some keys are absent.") except IOError: logger.error(f'Failed to write ${name} config.') @staticmethod - def _get_box_config(config_maps_dict, name, config_file_name, path): + def _get_box_config(config_maps_dict: Dict[str, Dict], + name: str, + config_file_name: str, + path: Path) -> None: try: if 'items' in config_maps_dict: for config_map in config_maps_dict['items']: diff --git a/th2_common/schema/filter/strategy/abstract_filter_strategy.py b/th2_common/schema/filter/strategy/abstract_filter_strategy.py new file mode 100644 index 0000000..76cf3e4 --- /dev/null +++ b/th2_common/schema/filter/strategy/abstract_filter_strategy.py @@ -0,0 +1,38 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC +from fnmatch import fnmatch +from typing import Callable, Dict + +from th2_common.schema.filter.strategy.filter_strategy import FilterStrategy +from th2_common.schema.message.configuration.message_configuration import FieldFilterConfiguration, FieldFilterOperation + + +class AbstractFilterStrategy(FilterStrategy, ABC): + + @staticmethod + def check_value(value: str, filter_configuration: FieldFilterConfiguration) -> bool: + expected = filter_configuration.value + + options: Dict[FieldFilterOperation, Callable[[str], bool]] = { + FieldFilterOperation.EQUAL: (lambda v: v == expected), + FieldFilterOperation.NOT_EQUAL: (lambda v: v != expected), + FieldFilterOperation.EMPTY: (lambda v: len(v) == 0), + FieldFilterOperation.NOT_EMPTY: (lambda v: len(v) != 0), + FieldFilterOperation.WILDCARD: (lambda v: fnmatch(v, expected)), # type: ignore + FieldFilterOperation.UNKNOWN: (lambda v: False) + } + + return options[filter_configuration.operation](value) diff --git a/th2_common/schema/filter/strategy/filter_strategy.py b/th2_common/schema/filter/strategy/filter_strategy.py index a33c14c..8db456b 100644 --- a/th2_common/schema/filter/strategy/filter_strategy.py +++ b/th2_common/schema/filter/strategy/filter_strategy.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,18 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. - from abc import ABC, abstractmethod -from typing import List - -from google.protobuf.message import Message - -from th2_common.schema.message.configuration.message_configuration import RouterFilterConfiguration +from typing import Any, Optional class FilterStrategy(ABC): @abstractmethod - def verify(self, message: Message, router_filter: RouterFilterConfiguration = None, - router_filters: List[RouterFilterConfiguration] = None): + def verify(self, + message: Any, + router_filters: Optional[Any] = None) -> bool: pass diff --git a/th2_common/schema/filter/strategy/impl/default_filter_strategy.py b/th2_common/schema/filter/strategy/impl/default_filter_strategy.py index f31776d..02d0f3f 100644 --- a/th2_common/schema/filter/strategy/impl/default_filter_strategy.py +++ b/th2_common/schema/filter/strategy/impl/default_filter_strategy.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,54 +12,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -from fnmatch import fnmatch -from typing import List +from typing import Dict, List, Optional, Union -from google.protobuf.message import Message - -from th2_common.schema.filter.strategy.filter_strategy import FilterStrategy +from th2_common.schema.filter.strategy.abstract_filter_strategy import AbstractFilterStrategy from th2_common.schema.message.configuration.message_configuration import FieldFilterConfiguration, \ - RouterFilterConfiguration -from th2_common.schema.strategy.field_extraction.impl.th2_batch_msg_field_extraction import Th2BatchMsgFieldExtraction + MqRouterFilterConfiguration +from th2_common.schema.strategy.field_extraction.th2_msg_field_extraction import Th2MsgFieldExtraction +from th2_grpc_common.common_pb2 import MessageGroup -class DefaultFilterStrategy(FilterStrategy): +class DefaultFilterStrategy(AbstractFilterStrategy): - def __init__(self, extract_strategy=Th2BatchMsgFieldExtraction()) -> None: - self.extract_strategy = extract_strategy + RouterFiltersType = Union[List[MqRouterFilterConfiguration], MqRouterFilterConfiguration] - def verify(self, message: Message, router_filter: RouterFilterConfiguration = None, - router_filters: List[RouterFilterConfiguration] = None): - if router_filters is None: - filters = router_filter.get_message() + router_filter.get_metadata() - return self.check_values(self.extract_strategy.get_fields(message), filters) - else: - if len(router_filters) == 0: - return True - for fields_filter in router_filters: - if self.verify(message=message, router_filter=fields_filter): - return True - return False + def __init__(self) -> None: + self.extract_strategy = Th2MsgFieldExtraction() - def check_values(self, message_fields: {str: str}, field_filters: List[FieldFilterConfiguration]) -> bool: - for field_filter in field_filters: - msg_field_value = message_fields[field_filter.field_name] - if not self.check_value(msg_field_value, field_filter): - return False - return True + def verify(self, + message: MessageGroup, + router_filters: Optional[RouterFiltersType] = None) -> bool: + if isinstance(router_filters, MqRouterFilterConfiguration): + filters = router_filters.message + router_filters.metadata + return any(self.check_values(self.extract_strategy.get_fields(any_message), filters) + for any_message in message.messages) - def check_value(self, value, filter_configuration: FieldFilterConfiguration): - expected = filter_configuration.value + elif isinstance(router_filters, list) and len(router_filters) > 0: + return any(self.verify(message, fields_filter) for fields_filter in router_filters) - if filter_configuration.operation.name == 'EQUAL': - return value == expected - elif filter_configuration.operation.name == 'NOT_EQUAL': - return value != expected - elif filter_configuration.operation.name == 'EMPTY': - return len(value) == 0 - elif filter_configuration.operation.name == 'NOT_EMPTY': - return len(value) != 0 - elif filter_configuration.operation.name == 'WILDCARD': - return fnmatch(value, expected) else: - return False + return True + + def check_values(self, message_fields: Dict[str, str], field_filters: List[FieldFilterConfiguration]) -> bool: + return all(self.check_value(message_fields[field_filter.field_name], field_filter) + if field_filter.field_name in message_fields else False + for field_filter in field_filters) diff --git a/th2_common/schema/filter/strategy/impl/default_grpc_filter_strategy.py b/th2_common/schema/filter/strategy/impl/default_grpc_filter_strategy.py new file mode 100644 index 0000000..e0c0bdd --- /dev/null +++ b/th2_common/schema/filter/strategy/impl/default_grpc_filter_strategy.py @@ -0,0 +1,36 @@ +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Optional, Union + +from th2_common.schema.filter.strategy.abstract_filter_strategy import AbstractFilterStrategy +from th2_common.schema.grpc.configuration.grpc_configuration import GrpcFilterConfiguration + + +class DefaultGrpcFilterStrategy(AbstractFilterStrategy): + + RouterFiltersType = Union[List[GrpcFilterConfiguration], GrpcFilterConfiguration] + + def verify(self, + message: Dict[str, str], + router_filters: Optional[RouterFiltersType] = None) -> bool: + if isinstance(router_filters, GrpcFilterConfiguration): + return all(self.check_value(message[field_filter.field_name], field_filter) + if field_filter.field_name in message else False + for field_filter in router_filters.properties) + + elif isinstance(router_filters, list) and len(router_filters) > 0: + return all(self.verify(message, fields_filter) for fields_filter in router_filters) + else: + return True diff --git a/th2_common/schema/grpc/configuration/grpc_configuration.py b/th2_common/schema/grpc/configuration/grpc_configuration.py index 3d015bc..f1091fd 100644 --- a/th2_common/schema/grpc/configuration/grpc_configuration.py +++ b/th2_common/schema/grpc/configuration/grpc_configuration.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,88 +13,105 @@ # limitations under the License. import json +from typing import Any, Dict, List, Optional, Tuple, Union from th2_common.schema.configuration.abstract_configuration import AbstractConfiguration +from th2_common.schema.message.configuration.message_configuration import FieldFilterConfiguration -from typing import List - -from th2_common.schema.message.configuration.message_configuration import FieldFilterConfiguration, \ - RouterFilterConfiguration - - -class GrpcRawRobinStrategy: - - def __init__(self, endpoints, name) -> None: - self.endpoints = endpoints - self.name = name - - -class GrpcServiceConfiguration(AbstractConfiguration): - pass +class GrpcConfiguration(AbstractConfiguration): -class GrpcServerConfiguration(AbstractConfiguration): + def __init__(self, + services: Dict[str, Dict[str, Any]], + server: Optional[Dict[str, Any]] = None, + **kwargs: Any) -> None: + self.services: Dict[str, GrpcServiceConfiguration] = { + name: GrpcServiceConfiguration(**params) for name, params in services.items() + } + if server is not None: + self.server = GrpcServerConfiguration(**server) - def __init__(self, attributes, host, port, workers, **kwargs) -> None: - self.attributes = attributes - self.host = host - self.port = port - self.workers = workers self.check_unexpected_args(kwargs) -class GrpcEndpointConfiguration(AbstractConfiguration): - - def __init__(self, host, port, attributes, **kwargs) -> None: - self.host = host - self.port = port - self.attributes = attributes - self.check_unexpected_args(kwargs) +class GrpcConnectionConfiguration(AbstractConfiguration): + def __init__(self, + workers: int = 5, + retryPolicy: Optional[Dict[str, Any]] = None, + request_size_limit: Union[int, float] = 4, + **kwargs: Any) -> None: + if retryPolicy is None: + retryPolicy = {} + self.workers = int(workers) + self.retry_policy = GrpcRetryPolicyConfiguration(**retryPolicy) + self.request_size_limit = [ + ('grpc.max_receive_message_length', round(request_size_limit * 1024 * 1024)), + ('grpc.max_send_message_length', round(request_size_limit * 1024 * 1024)) + ] -class GrpcConfiguration(AbstractConfiguration): - - def __init__(self, services, server=None, **kwargs) -> None: - self.services = services - if server is not None: - self.serverConfiguration = GrpcServerConfiguration(**server) self.check_unexpected_args(kwargs) -class GrpcRouterFilterConfiguration(RouterFilterConfiguration): - - def __init__(self, endpoint: str, metadata, message, **kwargs) -> None: - self.metadata = metadata - self.message = message - self.endpoint = endpoint - self.check_unexpected_args(kwargs) +class GrpcServiceConfiguration(AbstractConfiguration): - def get_metadata(self) -> List[FieldFilterConfiguration]: - return self.metadata + PropertiesType = List[Dict[str, str]] + FiltersType = List[Dict[str, PropertiesType]] + + def __init__(self, + endpoints: Dict[str, Any], + service_class: str, + attributes: Optional[List[str]] = None, + filters: Optional[FiltersType] = None, + strategy: Optional[Dict[str, Any]] = None, # deprecated + **kwargs: Any) -> None: + self.endpoints: Dict[str, GrpcEndpointConfiguration] = { + endpoint: GrpcEndpointConfiguration(**configuration) for endpoint, configuration in endpoints.items() + } + self.service_class = service_class - def get_message(self) -> List[FieldFilterConfiguration]: - return self.message + if attributes is not None: + self.attributes = attributes + else: + self.attributes = [] + if filters is not None: + self.filters = [GrpcFilterConfiguration(**filter_obj) for filter_obj in filters] + else: + self.filters = [] -class GrpcRawFilterStrategy: + self.check_unexpected_args(kwargs) - def __init__(self, filters) -> None: - self.filters = [GrpcRouterFilterConfiguration(**filter_configuration) for filter_configuration in filters] +class GrpcServerConfiguration(AbstractConfiguration): -class GrpcRouterConfiguration(AbstractConfiguration): + def __init__(self, + attributes: List[str], + host: str, + port: int, + workers: int, + **kwargs: Any) -> None: + self.attributes = attributes + self.host = host + self.port = port + self.workers = workers - def __init__(self, workers=5, retryPolicy=None, **kwargs): - if retryPolicy is None: - retryPolicy = dict() - self.workers = int(workers) - self.retry_policy = GrpcRetryPolicy(**retryPolicy) self.check_unexpected_args(kwargs) -class GrpcRetryPolicy: +class GrpcRetryPolicyConfiguration(AbstractConfiguration): + + OptionsType = Tuple[str, Union[str, int]] + ServicesDictType = Dict[str, str] - def __init__(self, maxAttemtps=5, initialBackoff=5., maxBackoff=60., backoffMultiplier=2, statusCodes=None, services=None): + def __init__(self, + maxAttemtps: int = 5, + initialBackoff: float = 5., + maxBackoff: float = 60., + backoffMultiplier: int = 2, + statusCodes: Optional[List[str]] = None, + services: Optional[List[ServicesDictType]] = None, + **kwargs: Any) -> None: """ Initializes retry policy for later usage of 'options' parameter. @@ -103,7 +120,8 @@ def __init__(self, maxAttemtps=5, initialBackoff=5., maxBackoff=60., backoffMult :param float maxBackoff: maximum delay before the retry (defaults to 60.0) :param int backoffMultiplier: multiplier by which every subsequent delay is changed (defaults to 2) :param list statusCodes: list of status code strings which invoke retry attempt (defaults to ['UNAVAILABLE']) - :param list services: list of dictionaries with keys 'service' and 'method', indicating where policy is applicable (defaults to every) + :param list services: list of dictionaries with keys 'service' and 'method', indicating where policy is + applicable (defaults to every) """ self.max_attempts = maxAttemtps self.initial_backoff = initialBackoff @@ -112,23 +130,70 @@ def __init__(self, maxAttemtps=5, initialBackoff=5., maxBackoff=60., backoffMult self.status_codes = statusCodes self.services = services + self.check_unexpected_args(kwargs) + @property - def options(self): + def options(self) -> List[OptionsType]: """Returns the retry options for the channel constructor.""" if self.services is None: self.services = [{}] + if self.status_codes is None: - self.status_codes = ["UNAVAILABLE"] - service_config_json = { - 'methodConfig': [{ - 'name': self.services, - 'retryPolicy': { - 'maxAttempts': self.max_attempts, - 'initialBackoff': f'{self.initial_backoff}s', - 'maxBackoff': f'{self.max_backoff}s', - 'backoffMultiplier': self.backoff_multiplier, - 'retryableStatusCodes': self.status_codes, - }, - }] + self.status_codes = ['UNAVAILABLE'] + + service_config_json = { # noqa: ECE001 + 'methodConfig': [ + { + 'name': self.services, + 'retryPolicy': { + 'maxAttempts': self.max_attempts, + 'initialBackoff': f'{self.initial_backoff}s', + 'maxBackoff': f'{self.max_backoff}s', + 'backoffMultiplier': self.backoff_multiplier, + 'retryableStatusCodes': self.status_codes, + } + } + ] } - return [("grpc.enable_retries", 1), ("grpc.service_config", json.dumps(service_config_json))] + + return [('grpc.enable_retries', 1), ('grpc.service_config', json.dumps(service_config_json))] + + +class GrpcEndpointConfiguration(AbstractConfiguration): + + def __init__(self, + host: str, + port: int, + attributes: Optional[List[str]] = None, + **kwargs: Any) -> None: + self.host = host + self.port = port + + if attributes is not None: + self.attributes = attributes + else: + self.attributes = [] + + self.check_unexpected_args(kwargs) + + +class GrpcFilterConfiguration(AbstractConfiguration): + + FilterType = Dict[str, str] + + def __init__(self, + properties: Optional[List[FilterType]] = None, + metadata: Optional[List[FilterType]] = None, + message: Optional[List[FilterType]] = None, + **kwargs: Any): + self.properties: List[FieldFilterConfiguration] = [] + + if properties is not None: + self.properties.extend([FieldFilterConfiguration(**prop) for prop in properties]) + else: + if metadata is not None: + self.properties.extend([FieldFilterConfiguration(**md) for md in metadata]) + if message is not None: + self.properties.extend([FieldFilterConfiguration(**msg) for msg in message]) + + self.check_unexpected_args(kwargs) diff --git a/th2_common/schema/grpc/router/abstract_grpc_router.py b/th2_common/schema/grpc/router/abstract_grpc_router.py index 51db523..c269c98 100644 --- a/th2_common/schema/grpc/router/abstract_grpc_router.py +++ b/th2_common/schema/grpc/router/abstract_grpc_router.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,32 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. - from abc import ABC from concurrent.futures.thread import ThreadPoolExecutor +from typing import Dict, List, Optional import grpc - -from th2_common.schema.grpc.configuration.grpc_configuration import GrpcConfiguration, GrpcRouterConfiguration +from th2_common.schema.grpc.configuration.grpc_configuration import GrpcConfiguration, GrpcConnectionConfiguration from th2_common.schema.grpc.router.grpc_router import GrpcRouter class AbstractGrpcRouter(GrpcRouter, ABC): - def __init__(self, grpc_configuration: GrpcConfiguration, - grpc_router_configuration: GrpcRouterConfiguration) -> None: + def __init__(self, + grpc_configuration: GrpcConfiguration, + grpc_router_configuration: GrpcConnectionConfiguration) -> None: self.grpc_configuration = grpc_configuration self.grpc_router_configuration = grpc_router_configuration - self.servers = [] - self.channels = {} + self.servers: List[grpc.Server] = [] + self.channels: Dict[str, grpc._channel.Channel] = {} - def __add_insecure_port(self, server): - if self.grpc_configuration.serverConfiguration.host is None: - server.add_insecure_port(f'[::]:{self.grpc_configuration.serverConfiguration.port}') + def __add_insecure_port(self, server: grpc.Server) -> None: + if self.grpc_configuration.server.host is None: + server.add_insecure_port(f'[::]:{self.grpc_configuration.server.port}') else: server.add_insecure_port( - f'{self.grpc_configuration.serverConfiguration.host}:{self.grpc_configuration.serverConfiguration.port}') - + f'{self.grpc_configuration.server.host}:{self.grpc_configuration.server.port}' + ) + @property def server(self) -> grpc.Server: """ @@ -46,12 +47,13 @@ def server(self) -> grpc.Server: Returns: grpc.Server: A server object. """ - server = grpc.server(ThreadPoolExecutor(max_workers=self.grpc_router_configuration.workers)) + server = grpc.server(ThreadPoolExecutor(max_workers=self.grpc_router_configuration.workers), + options=self.grpc_router_configuration.request_size_limit) self.__add_insecure_port(server) self.servers.append(server) return server - + @property def async_server(self) -> grpc.aio.Server: """ @@ -66,7 +68,7 @@ def async_server(self) -> grpc.aio.Server: return server - def close(self, grace=None): + def close(self, grace: Optional[int] = None) -> None: for server in self.servers: server.stop(grace) diff --git a/th2_common/schema/grpc/router/grpc_router.py b/th2_common/schema/grpc/router/grpc_router.py index 0067be4..b6c897e 100644 --- a/th2_common/schema/grpc/router/grpc_router.py +++ b/th2_common/schema/grpc/router/grpc_router.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. - from abc import ABC, abstractmethod +from typing import Callable import grpc @@ -21,7 +21,7 @@ class GrpcRouter(ABC): @abstractmethod - def get_service(self, cls): + def get_service(self, cls: Callable) -> Callable: pass @property @@ -33,3 +33,7 @@ def server(self) -> grpc.Server: @abstractmethod def async_server(self) -> grpc.aio.Server: pass + + @abstractmethod + def close(self) -> None: + pass diff --git a/th2_common/schema/grpc/router/impl/default_grpc_router.py b/th2_common/schema/grpc/router/impl/default_grpc_router.py index 01581cf..ca61bc3 100644 --- a/th2_common/schema/grpc/router/impl/default_grpc_router.py +++ b/th2_common/schema/grpc/router/impl/default_grpc_router.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,84 +12,106 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools +from typing import Any, Callable, Dict, List, Optional, Tuple -from importlib import import_module -from pathlib import Path -from pkgutil import iter_modules - +import google.protobuf.message import grpc - +from grpc import _channel from th2_common.schema.exception.grpc_router_error import GrpcRouterError -from th2_common.schema.grpc.configuration.grpc_configuration import GrpcConfiguration, GrpcRouterConfiguration, \ - GrpcRetryPolicy +from th2_common.schema.filter.strategy.impl.default_grpc_filter_strategy import DefaultGrpcFilterStrategy +from th2_common.schema.grpc.configuration.grpc_configuration import GrpcConfiguration, GrpcConnectionConfiguration, \ + GrpcEndpointConfiguration, GrpcServiceConfiguration from th2_common.schema.grpc.router.abstract_grpc_router import AbstractGrpcRouter -import th2_common.schema.strategy.route.impl as route class DefaultGrpcRouter(AbstractGrpcRouter): - def __init__(self, grpc_configuration: GrpcConfiguration, - grpc_router_configuration: GrpcRouterConfiguration) -> None: + def __init__(self, + grpc_configuration: GrpcConfiguration, + grpc_router_configuration: GrpcConnectionConfiguration) -> None: super().__init__(grpc_configuration, grpc_router_configuration) - self.strategies = dict() - self.__load_strategies() - def get_service(self, cls): - return cls(self) + def get_service(self, cls: Callable) -> Callable: + return cls(self) # type: ignore class Connection: - def __init__(self, service, strategy_obj, stub_class, channels, options): - self.service = service - self.strategy_obj = strategy_obj + def __init__(self, + services: List[GrpcServiceConfiguration], + stub_class: Callable, + channels: Dict[str, _channel.Channel], + options: List[Tuple[str, Any]]) -> None: + self.services = services self.stubClass = stub_class self.channels = channels self.options = options - self.stubs = {} + self.stubs: Dict[str, Callable] = {} + self._grpc_filter_strategy = DefaultGrpcFilterStrategy() + self._endpoint_generators: Dict[GrpcServiceConfiguration, itertools.cycle[str]] = {} - def __create_stub_if_not_exists(self, endpoint_name, config): - socket = f"{config['host']}:{config['port']}" + def __create_stub_if_not_exists(self, endpoint_name: str, config: GrpcEndpointConfiguration) -> None: + socket = f'{config.host}:{config.port}' if socket not in self.channels: self.channels[socket] = grpc.insecure_channel(socket, options=self.options) if endpoint_name not in self.stubs: self.stubs[endpoint_name] = self.stubClass(self.channels[socket]) - def create_request(self, request_name, request, timeout): - endpoint = self.strategy_obj.get_endpoint(request) - endpoint_config = self.service['endpoints'][endpoint] + def create_request(self, + request_name: str, + request: google.protobuf.message.Message, + timeout: int, + properties: Optional[Dict[str, str]] = None) -> Optional[google.protobuf.message.Message]: + service = self._filter_services(properties) + endpoint = self._get_next_endpoint(service) + endpoint_config = service.endpoints[endpoint] + if endpoint_config is not None: self.__create_stub_if_not_exists(endpoint, endpoint_config) stub = self.stubs[endpoint] - if stub is not None: - return getattr(stub, request_name)(request, timeout=timeout) - def get_connection(self, service_class, stub_class): - find_service = None - if self.grpc_configuration.services: - for service in self.grpc_configuration.services: - if self.grpc_configuration.services[service]['service-class'].split('.')[-1] == service_class.__name__: - find_service = self.grpc_configuration.services[service] - break - else: - raise GrpcRouterError("Services list are empty in 'grpc.json'. Check your links") + if stub is not None: + return getattr(stub, request_name)(request, timeout=timeout) # type: ignore - strategy_name = find_service['strategy']['name'] - strategy_class = self.strategies[strategy_name] - if strategy_class is None: return None - strategy_obj = strategy_class(find_service['strategy']) - return self.Connection(find_service, strategy_obj, stub_class, self.channels, self.grpc_router_configuration.retry_policy.options) - def __load_strategies(self): - package_dir = str(Path(route.__file__).resolve().parent) + def _filter_services(self, properties: Optional[Dict[str, str]]) -> GrpcServiceConfiguration: + if not properties: + services = self.services + else: + services = [ + service for service in self.services + if self._grpc_filter_strategy.verify(properties, router_filters=service.filters) + ] + + if len(services) != 1: + raise GrpcRouterError(f'Number of services matching properties should be 1, not {len(services)}. ' + 'Check your gRPC configuration') - for _, module_name, _ in iter_modules([package_dir]): - module = import_module(f'{route.__name__}.{module_name}') - for name in dir(module): - if not name.startswith('__'): - attr = getattr(module, name) - if 'get_endpoint' in dir(attr): - self.strategies[name.lower()] = attr + return services[0] + + def _get_next_endpoint(self, service: GrpcServiceConfiguration) -> str: + if service not in self._endpoint_generators: + self._endpoint_generators[service] = itertools.cycle(service.endpoints) + + return next(self._endpoint_generators[service]) # type: ignore + + def get_connection(self, service_class: Callable, stub_class: Callable) -> Optional[Connection]: + if self.grpc_configuration.services: + find_services = self._filter_services_by_name(service_class.__name__) + + if find_services: + return self.Connection(find_services, + stub_class, + self.channels, + self.grpc_router_configuration.retry_policy.options) + return None + else: + raise GrpcRouterError("Services list are empty in 'grpc.json'. Check your links") - self.strategies.pop('routingstrategy', None) + def _filter_services_by_name(self, service_class_name: str) -> List[GrpcServiceConfiguration]: + return list(filter( # noqa: ECE001 + lambda service_cfg: (service_cfg.service_class.split('.')[-1] == service_class_name), + self.grpc_configuration.services.values() + )) diff --git a/th2_common/schema/strategy/field_extraction/field_extraction_strategy.py b/th2_common/schema/log/handlers.py similarity index 50% rename from th2_common/schema/strategy/field_extraction/field_extraction_strategy.py rename to th2_common/schema/log/handlers.py index beda41e..6c3799a 100644 --- a/th2_common/schema/strategy/field_extraction/field_extraction_strategy.py +++ b/th2_common/schema/log/handlers.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging -from abc import ABC, abstractmethod +from prometheus_client import Counter -from google.protobuf.message import Message +class MetricLogHandler(logging.Handler): -class FieldExtractionStrategy(ABC): + def __init__(self, level: str = 'ERROR', metric_name: str = 'th2_logging_metric') -> None: + self.metric = Counter(metric_name, 'This metric registers log entries below specified level', + ['logger', 'level']) + super().__init__(level) - @abstractmethod - def get_fields(self, message: Message) -> {str: str}: - pass + def emit(self, record: logging.LogRecord) -> None: + self.metric.labels(record.name, record.levelname).inc() diff --git a/th2_common/schema/log/log4py.conf b/th2_common/schema/log/log4py.conf index f91d662..1b2a653 100644 --- a/th2_common/schema/log/log4py.conf +++ b/th2_common/schema/log/log4py.conf @@ -1,8 +1,8 @@ [loggers] -keys=root,th2_common,pika +keys=root,th2_common,aiopika [handlers] -keys=consoleHandler,fileHandler +keys=consoleHandler,fileHandler,metricsHandler [formatters] keys=formatter @@ -18,9 +18,9 @@ qualname=th2_common handlers=consoleHandler,fileHandler propagate=0 -[logger_pika] +[logger_aiopika] level=WARNING -qualname=pika +qualname=aio_pika handlers=consoleHandler,fileHandler propagate=0 diff --git a/th2_common/schema/log/log_config.json b/th2_common/schema/log/log_config.json new file mode 100644 index 0000000..a3d93b3 --- /dev/null +++ b/th2_common/schema/log/log_config.json @@ -0,0 +1,47 @@ +{ + "version": 1, + "root": + { + "level": "INFO", + "handlers": ["consoleHandler", "fileHandler"], + "propagate": 0 + }, + "loggers": + { + "th2_common": + { + "level": "INFO", + "handlers": ["consoleHandler", "fileHandler"], + "propagate": 0 + }, + "aio_pika": + { + "level": "WARNING", + "handlers": ["consoleHandler", "fileHandler"], + "propagate": 0 + } + }, + "handlers": + { + "consoleHandler": + { + "class": "logging.StreamHandler", + "formatter": "formatter", + "stream": "ext://sys.stdout" + }, + "fileHandler": + { + "class": "logging.FileHandler", + "formatter": "formatter", + "filename": "../all.log" + } + }, + "formatters": + { + "formatter": + { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + } + }, + "disable_existing_loggers": 0 +} diff --git a/th2_common/schema/log/trace.py b/th2_common/schema/log/trace.py index cc1c2a0..49f46a3 100644 --- a/th2_common/schema/log/trace.py +++ b/th2_common/schema/log/trace.py @@ -1,4 +1,4 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,23 +13,23 @@ # limitations under the License. import logging - +from typing import Any _trace_installed = False -def install_trace_logger(): +def install_trace_logger() -> None: global _trace_installed if _trace_installed: return TRACE = 5 - def trace(self, msg, *args, **kwargs): + def trace(self: logging.Logger, msg: str, *args: Any, **kwargs: Any) -> None: if self.isEnabledFor(TRACE): self._log(TRACE, msg, args, **kwargs) - def log_to_root(msg, *args, **kwargs): + def log_to_root(msg: str, *args: Any, **kwargs: Any) -> None: logging.log(TRACE, msg, *args, **kwargs) logging.addLevelName(TRACE, 'TRACE') diff --git a/th2_common/schema/message/configuration/message_configuration.py b/th2_common/schema/message/configuration/message_configuration.py index 2cc3f5d..be41e94 100644 --- a/th2_common/schema/message/configuration/message_configuration.py +++ b/th2_common/schema/message/configuration/message_configuration.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,77 +12,78 @@ # See the License for the specific language governing permissions and # limitations under the License. -from enum import Enum, auto +from enum import Enum +from typing import Any, Dict, List, Optional, Set from th2_common.schema.configuration.abstract_configuration import AbstractConfiguration -from abc import ABC, abstractmethod -from typing import List - class FieldFilterOperation(Enum): - EQUAL = auto() - NOT_EQUAL = auto() - EMPTY = auto() - NOT_EMPTY = auto() - WILDCARD = auto() - - -class FieldFilterConfiguration(AbstractConfiguration): + EQUAL = 'EQUAL' + NOT_EQUAL = 'NOT_EQUAL' + EMPTY = 'EMPTY' + NOT_EMPTY = 'NOT_EMPTY' + WILDCARD = 'WILDCARD' + UNKNOWN = 'UNKNOWN' - def __init__(self, value: str = None, expectedValue: str = None, fieldName: str = None, - operation: FieldFilterOperation = None, **kwargs) -> None: - self.value = value or expectedValue - self.field_name = fieldName - self.operation = FieldFilterOperation[operation] - self.check_unexpected_args(kwargs) + @classmethod + def _missing_(cls, value: object) -> Any: + return FieldFilterOperation.UNKNOWN - def __str__(self): - return f'Value: {self.value} Field name: {self.field_name} Operation: {self.operation}' +class MessageRouterConfiguration(AbstractConfiguration): -class RouterFilterConfiguration(AbstractConfiguration, ABC): - - @abstractmethod - def get_metadata(self) -> List[FieldFilterConfiguration]: - pass - - @abstractmethod - def get_message(self) -> List[FieldFilterConfiguration]: - pass + def __init__(self, queues: Dict[str, Any], **kwargs: Any) -> None: + self.queues = {queue_alias: QueueConfiguration(**queues[queue_alias]) for queue_alias in queues} + self.check_unexpected_args(kwargs) -class MqRouterFilterConfiguration(RouterFilterConfiguration): + def get_queue_by_alias(self, queue_alias: str) -> 'QueueConfiguration': + return self.queues[queue_alias] - def __init__(self, metadata=None, message=None, **kwargs) -> None: + def find_queues_by_attr(self, attrs: Set[str]) -> Dict[str, 'QueueConfiguration']: + result = {} + for queue_alias in self.queues: + if all(attr in self.queues[queue_alias].attributes for attr in attrs): + result[queue_alias] = self.queues[queue_alias] + return result - self.metadata = [] - self.message = [] - if isinstance(metadata, dict) and isinstance(message, dict): - self.metadata = [FieldFilterConfiguration(**metadata[key], fieldName=key) for key in metadata.keys()] - self.message = [FieldFilterConfiguration(**message[key], fieldName=key) for key in message.keys()] +class MqConnectionConfiguration(AbstractConfiguration): - elif isinstance(metadata, list) and isinstance(message, list): - self.metadata = [FieldFilterConfiguration(**key) for key in metadata] - self.message = [FieldFilterConfiguration(**key) for key in message] + def __init__(self, + subscriberName: Optional[str] = None, + connectionTimeout: int = -1, + connectionCloseTimeout: int = 10000, + maxRecoveryAttempts: int = 5, + minConnectionRecoveryTimeout: int = 10000, + maxConnectionRecoveryTimeout: int = 60000, + prefetchCount: int = 10, + messageRecursionLimit: int = 100, + **kwargs: Any) -> None: + self.subscriber_name = subscriberName + self.connection_timeout = int(connectionTimeout) + self.connection_close_timeout = int(connectionCloseTimeout) + self.max_recovery_attempts = int(maxRecoveryAttempts) + self.min_connection_recovery_timeout = int(minConnectionRecoveryTimeout) + self.max_connection_recovery_timeout = int(maxConnectionRecoveryTimeout) + self.prefetch_count = int(prefetchCount) + self.message_recursion_limit = int(messageRecursionLimit) self.check_unexpected_args(kwargs) - def get_metadata(self) -> List[FieldFilterConfiguration]: - return self.metadata - - def get_message(self) -> List[FieldFilterConfiguration]: - return self.message - - def __str__(self): - return self.metadata + self.message - class QueueConfiguration(AbstractConfiguration): - def __init__(self, name: str, queue: str, exchange: str, attributes: list, filters: list, can_read=True, - can_write=True, **kwargs) -> None: + def __init__(self, + name: str, + queue: str, + exchange: str, + attributes: List[str], + filters: List[Dict[str, Any]], + can_read: bool = True, + can_write: bool = True, + **kwargs: Any) -> None: self.routing_key = name self.queue = queue self.exchange = exchange @@ -90,41 +91,48 @@ def __init__(self, name: str, queue: str, exchange: str, attributes: list, filte self.filters = [MqRouterFilterConfiguration(**filter_schema) for filter_schema in filters] self.can_read = can_read self.can_write = can_write + self.check_unexpected_args(kwargs) -class MessageRouterConfiguration(AbstractConfiguration): - def __init__(self, queues: dict, **kwargs) -> None: - self.queues = {queue_alias: QueueConfiguration(**queues[queue_alias]) for queue_alias in queues.keys()} +class MqRouterFilterConfiguration(AbstractConfiguration): + + def __init__(self, + metadata: Optional[Dict[str, Any]] = None, + message: Optional[Dict[str, Any]] = None, + **kwargs: Any) -> None: + self.metadata = [] + self.message = [] + + if isinstance(metadata, dict): + self.metadata = [FieldFilterConfiguration(**metadata[key], fieldName=key) for key in metadata] + elif isinstance(metadata, list): + self.metadata = [FieldFilterConfiguration(**key) for key in metadata] + + if isinstance(message, dict): + self.message = [FieldFilterConfiguration(**message[key], fieldName=key) for key in message] + elif isinstance(message, list): + self.message = [FieldFilterConfiguration(**key) for key in message] + self.check_unexpected_args(kwargs) - def get_queue_by_alias(self, queue_alias): - return self.queues[queue_alias] + def __str__(self) -> str: + return str(self.metadata + self.message) - def find_queues_by_attr(self, attrs) -> {str: QueueConfiguration}: - result = dict() - for queue_alias in self.queues.keys(): - if all(attr in self.queues[queue_alias].attributes for attr in attrs): - result[queue_alias] = self.queues[queue_alias] - return result +class FieldFilterConfiguration(AbstractConfiguration): + + def __init__(self, + fieldName: str, + operation: str, + value: Optional[str] = None, + expectedValue: Optional[str] = None, + **kwargs: Any) -> None: + self.value = value or expectedValue + self.field_name = fieldName + self.operation = FieldFilterOperation(operation) -class ConnectionManagerConfiguration(AbstractConfiguration): - def __init__(self, subscriberName=None, - connectionTimeout=-1, - connectionCloseTimeout = 10000, - maxRecoveryAttempts=5, - minConnectionRecoveryTimeout=10000, - maxConnectionRecoveryTimeout=60000, - prefetchCount=10, - messageRecursionLimit=100, - **kwargs): - self.subscriber_name = subscriberName - self.connection_timeout = int(connectionTimeout) - self.connection_close_timeout = int(connectionCloseTimeout) - self.max_recovery_attempts = int(maxRecoveryAttempts) - self.min_connection_recovery_timeout = int(minConnectionRecoveryTimeout) - self.max_connection_recovery_timeout = int(maxConnectionRecoveryTimeout) - self.prefetch_count = int(prefetchCount) - self.message_recursion_limit = int(messageRecursionLimit) self.check_unexpected_args(kwargs) + + def __str__(self) -> str: + return f'Value: {self.value} Field name: {self.field_name} Operation: {self.operation}' diff --git a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_batch_subscriber.py b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_batch_subscriber.py index be74c52..00ecc95 100644 --- a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_batch_subscriber.py +++ b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_batch_subscriber.py @@ -14,17 +14,22 @@ from abc import ABC, abstractmethod -from th2_grpc_common.common_pb2 import Direction - +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer from th2_common.schema.filter.strategy.impl.default_filter_strategy import DefaultFilterStrategy from th2_common.schema.message.configuration.message_configuration import QueueConfiguration from th2_common.schema.message.impl.rabbitmq.abstract_rabbit_subscriber import AbstractRabbitSubscriber +from th2_common.schema.message.impl.rabbitmq.configuration.subscribe_target import SubscribeTarget from th2_common.schema.message.impl.rabbitmq.connection.connection_manager import ConnectionManager +from th2_grpc_common.common_pb2 import Direction, MessageGroupBatch class Metadata: - def __init__(self, sequence, message_type: str, direction: Direction, session_alias: str) -> None: + def __init__(self, + sequence: str, + message_type: str, + direction: Direction, + session_alias: str) -> None: self.sequence = sequence self.message_type = message_type self.direction = direction @@ -33,21 +38,27 @@ def __init__(self, sequence, message_type: str, direction: Direction, session_al class AbstractRabbitBatchSubscriber(AbstractRabbitSubscriber, ABC): - def __init__(self, connection_manager: ConnectionManager, queue_configuration: QueueConfiguration, - filter_strategy=DefaultFilterStrategy(), *subscribe_targets, th2_pin='') -> None: - super().__init__(connection_manager, queue_configuration, *subscribe_targets, th2_pin=th2_pin) + def __init__(self, + connection_manager: ConnectionManager, + subscribe_target: SubscribeTarget, + queue_configuration: QueueConfiguration, + filter_strategy: DefaultFilterStrategy, + th2_pin: str = '') -> None: + super().__init__(connection_manager, subscribe_target, queue_configuration, th2_pin=th2_pin) self.filters = queue_configuration.filters self.filter_strategy = filter_strategy - def filter(self, batch) -> bool: - messages = [msg for msg in self.get_messages(batch) if - self.filter_strategy.verify(message=msg, router_filters=self.filters)] + def filter(self, batch: MessageGroupBatch) -> bool: # noqa: A003 + messages = [ + message_group for message_group in self.get_messages(batch) if + self.filter_strategy.verify(message=message_group, router_filters=self.filters) + ] return len(messages) > 0 @abstractmethod - def get_messages(self, batch) -> list: + def get_messages(self, batch: MessageGroupBatch) -> RepeatedCompositeFieldContainer: pass @abstractmethod - def extract_metadata(self, message) -> Metadata: + def extract_metadata(self, message: MessageGroupBatch) -> Metadata: pass diff --git a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_message_router.py b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_message_router.py index 506c5b2..08f2312 100644 --- a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_message_router.py +++ b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_message_router.py @@ -15,14 +15,13 @@ from abc import ABC, abstractmethod from collections import defaultdict from threading import Lock -from typing import Callable - -from th2_grpc_common.common_pb2 import MessageGroupBatch, Message +from typing import Any, Callable, Dict, List, Optional, Set +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer from th2_common.schema.exception.router_error import RouterError from th2_common.schema.filter.strategy.filter_strategy import FilterStrategy from th2_common.schema.filter.strategy.impl.default_filter_strategy import DefaultFilterStrategy -from th2_common.schema.message.configuration.message_configuration import QueueConfiguration +from th2_common.schema.message.configuration.message_configuration import MessageRouterConfiguration, QueueConfiguration from th2_common.schema.message.impl.rabbitmq.connection.connection_manager import ConnectionManager from th2_common.schema.message.message_listener import MessageListener from th2_common.schema.message.message_router import MessageRouter @@ -30,15 +29,19 @@ from th2_common.schema.message.message_subscriber import MessageSubscriber from th2_common.schema.message.subscriber_monitor import SubscriberMonitor from th2_common.schema.util.util import get_debug_string_group, get_filters +from th2_grpc_common.common_pb2 import MessageGroupBatch class SubscriberMonitorImpl(SubscriberMonitor): - def __init__(self, subscriber: MessageSubscriber, lock=Lock()) -> None: - self.lock = lock + def __init__(self, subscriber: MessageSubscriber, lock: Optional[Lock] = None) -> None: + if lock is None: + self.lock = Lock() + else: + self.lock = lock self.subscriber = subscriber - def unsubscribe(self): + def unsubscribe(self) -> None: with self.lock: self.subscriber.close() @@ -48,38 +51,40 @@ class MultiplySubscribeMonitorImpl(SubscriberMonitor): def __init__(self, subscriber_monitors: list) -> None: self.subscriber_monitors = subscriber_monitors - def unsubscribe(self): + def unsubscribe(self) -> None: for monitor in self.subscriber_monitors: monitor.unsubscribe() class AbstractRabbitMessageRouter(MessageRouter, ABC): - def __init__(self, connection_manager, configuration) -> None: + def __init__(self, connection_manager: ConnectionManager, configuration: MessageRouterConfiguration) -> None: super().__init__(connection_manager, configuration) - self._filter_strategy = DefaultFilterStrategy() - self.queue_connections = list() # List of queue aliases, which configurations we used to create senders/subs. + self._filter_strategy: FilterStrategy = DefaultFilterStrategy() + # List of queue aliases, which configurations we used to create senders/subs. + self.queue_connections: List[str] = [] self.queue_connections_lock = Lock() - self.subscribers = dict() # queue_alias: subscriber-like objects. - self.senders = dict() + # queue_alias: subscriber-like objects. + self.subscribers: Dict[str, MessageSubscriber] = {} + self.senders: Dict[str, MessageSender] = {} @property @abstractmethod - def required_subscribe_attributes(self): + def required_subscribe_attributes(self) -> Set[str]: pass @property @abstractmethod - def required_send_attributes(self): + def required_send_attributes(self) -> Set[str]: pass - def add_subscribe_attributes(self, queue_attr): + def add_subscribe_attributes(self, *queue_attr: str) -> Set[str]: return self.required_subscribe_attributes.union(queue_attr) - def add_send_attributes(self, queue_attr): + def add_send_attributes(self, *queue_attr: str) -> Set[str]: return self.required_send_attributes.union(queue_attr) - def _subscribe_by_alias(self, callback: MessageListener, queue_alias) -> SubscriberMonitor: + def _subscribe_by_alias(self, callback: MessageListener, queue_alias: str) -> SubscriberMonitor: subscriber: MessageSubscriber = self.get_subscriber(queue_alias) subscriber.add_listener(callback) try: @@ -88,122 +93,143 @@ def _subscribe_by_alias(self, callback: MessageListener, queue_alias) -> Subscri raise RuntimeError('Can not start subscriber', e) return SubscriberMonitorImpl(subscriber, Lock()) - def subscribe(self, callback: MessageListener, *queue_attr) -> SubscriberMonitor: - attrs = self.add_subscribe_attributes(queue_attr) + def subscribe(self, callback: MessageListener, *queue_attr: str) -> SubscriberMonitor: + attrs = self.add_subscribe_attributes(*queue_attr) queues = self.configuration.find_queues_by_attr(attrs) + if len(queues) != 1: raise RouterError( f'Wrong amount of queues for subscribe. ' f'Found {len(queues)} queues, but must be only 1. Search was done by {queue_attr} attributes') + return self._subscribe_by_alias(callback, next(iter(queues.keys()))) - def subscribe_all(self, callback: MessageListener, *queue_attr) -> SubscriberMonitor: - attrs = self.add_subscribe_attributes(queue_attr) + def subscribe_all(self, callback: MessageListener, *queue_attr: str) -> SubscriberMonitor: + attrs = self.add_subscribe_attributes(*queue_attr) subscribers = [] - for queue_alias in self.configuration.find_queues_by_attr(attrs).keys(): + + for queue_alias in self.configuration.find_queues_by_attr(attrs): subscribers.append(self._subscribe_by_alias(callback, queue_alias)) + if len(subscribers) == 0: raise RouterError(f'Wrong amount of queues for subscribe_all. Must not be empty. ' f'Search was done by {attrs} attributes') return MultiplySubscribeMonitorImpl(subscribers) - def unsubscribe_all(self): + def unsubscribe_all(self) -> None: with self.queue_connections_lock: for queue in self.queue_connections: self.close_connection(queue) self.queue_connections.clear() - def close(self): + def close(self) -> None: self.unsubscribe_all() - def send(self, message, *queue_attr): - attrs = self.add_send_attributes(queue_attr) - self.filter_and_send(message, attrs, lambda x: None if len(x) == 1 else Exception(f'Found incorrect number of ' - f'pins {list(x.keys())} to the send' - f'operation by attributes ' - f'{attrs} and filters, ' - f'expected 1, actual {len(x)}. ' - f'Message: {get_debug_string_group(message)}.' - f'Filters: {get_filters(self.configuration, x.keys())}')) - - def send_all(self, message, *queue_attr): - attrs = self.add_send_attributes(queue_attr) - self.filter_and_send(message, attrs, lambda x: None if len(x) != 0 else Exception(f'Found incorrect number of ' - f'pins {list(x.keys())} to the send_all ' - f'operation by attributes ' - f'{attrs} and filters, ' - f'expected non-zero, actual {len(x)}. ' - f'Message: {get_debug_string_group(message)}.' - f'Filters: {get_filters(self.configuration, x.keys())}')) - - def filter_and_send(self, message, attrs, check: Callable): + def send(self, message: Any, *queue_attr: str) -> None: + attrs = self.add_send_attributes(*queue_attr) + self.filter_and_send(message, + attrs, + lambda x: (None if len(x) == 1 + else Exception(f'Found incorrect number of ' + f'pins {list(x.keys())} to the send' + f'operation by attributes ' + f'{attrs} and filters, ' + f'expected 1, actual {len(x)}. ' + f'Message: {get_debug_string_group(message)}.' + f'Filters: {get_filters(self.configuration, x.keys())}'))) + + def send_all(self, message: Any, *queue_attr: str) -> None: + attrs = self.add_send_attributes(*queue_attr) + self.filter_and_send(message, + attrs, + lambda x: (None if len(x) != 0 + else Exception(f'Found incorrect number of ' + f'pins {list(x.keys())} to the send_all ' + f'operation by attributes ' + f'{attrs} and filters, ' + f'expected non-zero, actual {len(x)}. ' + f'Message: {get_debug_string_group(message)}.' + f'Filters: {get_filters(self.configuration, x.keys())}'))) + + def filter_and_send(self, message: Any, attrs: Set[str], check: Callable) -> None: aliases_found_by_attrs = self.configuration.find_queues_by_attr(attrs) aliases_to_messages = self.split_and_filter(aliases_found_by_attrs, message) result_check = check(aliases_to_messages) + if result_check is not None: raise result_check + for alias, message in aliases_to_messages.items(): try: sender = self.get_sender(alias) sender.start() sender.send(message) + except Exception: raise RouterError('Can not start sender') - def split_and_filter(self, queue_aliases_to_configs, batch) -> dict: - result = defaultdict(MessageGroupBatch) - for message in self._get_messages(batch): + def split_and_filter(self, queue_aliases_to_configs: Dict[str, QueueConfiguration], batch: Any) -> Dict[str, Any]: + result: Dict[str, MessageGroupBatch] = defaultdict(MessageGroupBatch) + + for message_group in self._get_messages(batch): dropped_on_aliases = set() - aliases_suitable_for_message_part = set() + for queue_alias, queue_config in queue_aliases_to_configs.items(): filters = queue_config.filters - if len(filters) == 0 or self._filter_strategy.verify(message, router_filters=filters): - aliases_suitable_for_message_part.add(queue_alias) + if self._filter_strategy.verify(message_group, router_filters=filters): + self._add_message(result[queue_alias], message_group) else: dropped_on_aliases.add(queue_alias) - for queue_alias in aliases_suitable_for_message_part: - self._add_message(result[queue_alias], message) - self.update_dropped_metrics(MessageGroupBatch(groups=[message]), dropped_on_aliases) + + self.update_dropped_metrics(MessageGroupBatch(groups=[message_group]), *dropped_on_aliases) + return result @property - def filter_strategy(self): + def filter_strategy(self) -> FilterStrategy: return self._filter_strategy @filter_strategy.setter - def filter_strategy(self, filter_strategy: FilterStrategy): + def filter_strategy(self, filter_strategy: FilterStrategy) -> None: self._filter_strategy = filter_strategy - def get_subscriber(self, queue_alias) -> MessageSubscriber: + def get_subscriber(self, queue_alias: str) -> MessageSubscriber: queue_configuration = self.configuration.get_queue_by_alias(queue_alias) + with self.queue_connections_lock: if queue_alias not in self.queue_connections: self.queue_connections.append(queue_alias) + if not queue_configuration.can_read: raise RouterError('Reading from this queue is not allowed') + with self.subscriber_lock: if queue_alias not in self.subscribers or self.subscribers[queue_alias].is_close(): self.subscribers[queue_alias] = self.create_subscriber(self.connection_manager, queue_configuration, th2_pin=queue_alias) return self.subscribers[queue_alias] - def get_sender(self, queue_alias) -> MessageSender: + def get_sender(self, queue_alias: str) -> MessageSender: queue_configuration = self.configuration.get_queue_by_alias(queue_alias) + with self.queue_connections_lock: if queue_alias not in self.queue_connections: self.queue_connections.append(queue_alias) + if not queue_configuration.can_write: raise RouterError('Writing to this queue is not allowed') + with self.sender_lock: if queue_alias not in self.senders or self.senders[queue_alias].is_close(): self.senders[queue_alias] = self.create_sender(self.connection_manager, queue_configuration, th2_pin=queue_alias) return self.senders[queue_alias] - def close_connection(self, queue_alias): + def close_connection(self, queue_alias: str) -> None: with self.subscriber_lock: if queue_alias in self.subscribers and not self.subscribers[queue_alias].is_close(): self.subscribers[queue_alias].close() + with self.sender_lock: if queue_alias in self.senders and not self.senders[queue_alias].is_close(): self.senders[queue_alias].close() @@ -219,17 +245,17 @@ def create_subscriber(self, connection_manager: ConnectionManager, pass @abstractmethod - def _get_messages(self, batch) -> list: + def _get_messages(self, batch: Any) -> RepeatedCompositeFieldContainer: pass @abstractmethod - def _create_batch(self): + def _create_batch(self) -> Any: pass @abstractmethod - def _add_message(self, batch, message): + def _add_message(self, batch: Any, message: Any) -> None: pass @abstractmethod - def update_dropped_metrics(self, batch, pin): + def update_dropped_metrics(self, batch: Any, *pins: str) -> None: pass diff --git a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_sender.py b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_sender.py index fac9097..316ad16 100644 --- a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_sender.py +++ b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_sender.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from abc import ABC, abstractmethod +import logging +from typing import Any from prometheus_client import Counter - from th2_common.schema.message.impl.rabbitmq.connection.connection_manager import ConnectionManager from th2_common.schema.message.impl.rabbitmq.connection.publisher import Publisher from th2_common.schema.message.message_sender import MessageSender @@ -31,19 +31,23 @@ class AbstractRabbitSender(MessageSender, ABC): 'Amount of bytes sent', common_metrics.SENDER_LABELS) OUTGOING_MSG_QUANTITY_ABSTRACT = Counter('th2_rabbitmq_message_publish_total', - 'Amount of batches sent', - common_metrics.SENDER_LABELS) + 'Amount of batches sent', + common_metrics.SENDER_LABELS) _TH2_TYPE = 'unknown' - def __init__(self, connection_manager: ConnectionManager, exchange_name: str, send_queue: str, th2_pin='') -> None: + def __init__(self, + connection_manager: ConnectionManager, + exchange_name: str, + send_queue: str, + th2_pin: str = '') -> None: self.__publisher: Publisher = connection_manager.publisher self.__exchange_name: str = exchange_name self.__send_queue: str = send_queue self.__closed = True self.th2_pin = th2_pin - def start(self): + def start(self) -> None: if self.__send_queue is None or self.__exchange_name is None: raise Exception('Sender can not start. Sender did not init') self.__closed = False @@ -51,13 +55,15 @@ def start(self): def is_close(self) -> bool: return self.__closed - def close(self): + def close(self) -> None: self.__closed = True - def send(self, message): + def send(self, message: Any) -> None: labels = self.th2_pin, self._TH2_TYPE, self.__exchange_name, self.__send_queue + if message is None: raise ValueError('Value for send can not be null') + try: byted_message = self.value_to_bytes(message) self.__publisher.publish_message(exchange_name=self.__exchange_name, @@ -67,27 +73,28 @@ def send(self, message): self.OUTGOING_MSG_QUANTITY_ABSTRACT.labels(*labels).inc() self.OUTGOING_MSG_SIZE.labels(*labels).inc(len(byted_message)) - if logger.isEnabledFor(logging.TRACE): - logger.trace(f'Sending to exchange_name = "{self.__exchange_name}", ' - f'routing_key = "{self.__send_queue}", ' - f'message = {self.to_trace_string(message)}') + if logger.isEnabledFor(logging.TRACE): # type: ignore + logger.trace('Sending to exchange_name = "%s", ' # type: ignore + 'routing_key = "%s", ' + 'message = %s' + % (self.__exchange_name, self.__send_queue, self.to_trace_string(message))) elif logger.isEnabledFor(logging.DEBUG): - logger.debug(f'Sending to exchange_name = "{self.__exchange_name}", ' - f'routing_key = "{self.__send_queue}", ' - f'message = {self.to_debug_string(message)}') - - except Exception: - logger.exception('Can not send') + logger.debug('Sending to exchange_name = "%s", ' + 'routing_key = "%s", ' + 'message = %s}' + % (self.__exchange_name, self.__send_queue, self.to_debug_string(message))) + except Exception as e: + logger.exception(f"Can't send: {e}") @staticmethod @abstractmethod - def value_to_bytes(value): + def value_to_bytes(value: Any) -> bytes: pass @abstractmethod - def to_trace_string(self, value): + def to_trace_string(self, value: Any) -> str: pass @abstractmethod - def to_debug_string(self, value): + def to_debug_string(self, value: Any) -> str: pass diff --git a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_subscriber.py b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_subscriber.py index 34a51ff..faba41f 100644 --- a/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_subscriber.py +++ b/th2_common/schema/message/impl/rabbitmq/abstract_rabbit_subscriber.py @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import logging -import time from abc import ABC, abstractmethod +import logging from threading import Lock +import time +from typing import Any, Optional, Set import aio_pika from google.protobuf.message import DecodeError -from prometheus_client import Histogram, Counter - +from prometheus_client import Counter, Histogram from th2_common.schema.message.configuration.message_configuration import QueueConfiguration from th2_common.schema.message.impl.rabbitmq.configuration.subscribe_target import SubscribeTarget from th2_common.schema.message.impl.rabbitmq.connection.connection_manager import ConnectionManager @@ -29,6 +28,7 @@ from th2_common.schema.message.message_listener import MessageListener from th2_common.schema.message.message_subscriber import MessageSubscriber import th2_common.schema.metrics.common_metrics as common_metrics +from th2_grpc_common.common_pb2 import Message logger = logging.getLogger(__name__) @@ -40,53 +40,59 @@ class AbstractRabbitSubscriber(MessageSubscriber, ABC): 'Amount of bytes received', common_metrics.SUBSCRIBER_LABELS) HANDLING_DURATION = Histogram('th2_rabbitmq_message_process_duration_seconds', - 'Duration of one subscriber\'s handling process', + "Subscriber's handling process duration", common_metrics.SUBSCRIBER_LABELS, buckets=common_metrics.DEFAULT_BUCKETS) _th2_type = 'unknown' - def __init__(self, connection_manager: ConnectionManager, queue_configuration: QueueConfiguration, - subscribe_target: SubscribeTarget, th2_pin='') -> None: + def __init__(self, + connection_manager: ConnectionManager, + subscribe_target: SubscribeTarget, + queue_configuration: QueueConfiguration, + th2_pin: str = '') -> None: self.__subscribe_target = subscribe_target self.__attributes = tuple(set(queue_configuration.attributes)) self.th2_pin = th2_pin - self.listeners = set() + self.listeners: Set[MessageListener] = set() self.__lock_listeners = Lock() self.__consumer: Consumer = connection_manager.consumer - self.__consumer_tag = None + self.__consumer_tag: Optional[str] = None self.__closed = True self.__metrics = common_metrics.HealthMetrics(self) - def start(self): + def start(self) -> None: if self.__subscribe_target is None: raise Exception('Subscriber did not init') if self.__consumer_tag is None: queue = self.__subscribe_target.get_queue() self.__consumer_tag = self.__consumer.add_subscriber(queue_name=queue, - on_message_callback=self.handle) + on_message_callback=self.handle) # type: ignore self.__closed = False self.__metrics.enable() - async def handle(self, message: aio_pika.IncomingMessage): + async def handle(self, message: aio_pika.IncomingMessage) -> None: start_time = time.time() labels = self.th2_pin, self._th2_type, self.__subscribe_target.get_queue() + try: value = self.value_from_bytes(message.body) self.INCOMING_MESSAGE_SIZE.labels(*labels).inc(len(message.body)) + if value is None: raise ValueError('Received value is null') self.update_total_metrics(value) - if logger.isEnabledFor(logging.TRACE): - logger.trace(f'Received message: {self.to_trace_string(value)}') + + if logger.isEnabledFor(logging.TRACE): # type: ignore + logger.trace('Received message: %s' % self.to_trace_string(value)) # type: ignore elif logger.isEnabledFor(logging.DEBUG): - logger.debug(f'Received message: {self.to_debug_string(value)}') + logger.debug('Received message: %s' % self.to_debug_string(value)) if not self.filter(value): self.update_dropped_metrics(value) @@ -97,18 +103,19 @@ async def handle(self, message: aio_pika.IncomingMessage): except DecodeError as e: logger.exception( f'Can not parse value from delivery for: {message.consumer_tag} due to DecodeError: {e}\n' - f' body: {message.body}\n' + f' body: {message.body!r}\n' f' self: {self}\n') return + except Exception as e: logger.error(f'Can not parse value from delivery for: {message.consumer_tag}', e) return - finally: - self.HANDLING_DURATION.labels(*labels).observe(time.time()-start_time) + finally: + self.HANDLING_DURATION.labels(*labels).observe(time.time() - start_time) await message.ack() - def handle_with_listener(self, value): + def handle_with_listener(self, value: Message) -> None: with self.__lock_listeners: for listener in self.listeners: try: @@ -116,7 +123,7 @@ def handle_with_listener(self, value): except Exception as e: logger.warning(f"Message listener from class '{type(listener)}' threw exception {e}") - def add_listener(self, message_listener: MessageListener): + def add_listener(self, message_listener: MessageListener) -> None: if message_listener is None: return with self.__lock_listeners: @@ -125,37 +132,39 @@ def add_listener(self, message_listener: MessageListener): def is_close(self) -> bool: return self.__closed - def close(self): + def close(self) -> None: with self.__lock_listeners: for listener in self.listeners: listener.on_close() self.listeners.clear() - self.__consumer.remove_subscriber(self.__consumer_tag) - self.__closed = True + if self.__consumer_tag is not None: + self.__consumer.remove_subscriber(self.__consumer_tag) + + self.__closed = True self.__metrics.disable() @staticmethod @abstractmethod - def value_from_bytes(body): + def value_from_bytes(body: bytes) -> Any: pass - @abstractmethod - def filter(self, value) -> bool: + @abstractmethod # noqa: A003 + def filter(self, value: Any) -> bool: # noqa: A003 pass @abstractmethod - def to_trace_string(self, value): + def to_trace_string(self, value: Any) -> str: pass @abstractmethod - def to_debug_string(self, value): + def to_debug_string(self, value: Any) -> str: pass @abstractmethod - def update_dropped_metrics(self, batch): + def update_dropped_metrics(self, batch: Any) -> None: pass @abstractmethod - def update_total_metrics(self, batch): + def update_total_metrics(self, batch: Any) -> None: pass diff --git a/th2_common/schema/message/impl/rabbitmq/byte/__init__.py b/th2_common/schema/message/impl/rabbitmq/byte/__init__.py new file mode 100644 index 0000000..7d295a8 --- /dev/null +++ b/th2_common/schema/message/impl/rabbitmq/byte/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/th2_common/schema/message/impl/rabbitmq/byte/abstract_rabbit_byte_batch_router.py b/th2_common/schema/message/impl/rabbitmq/byte/abstract_rabbit_byte_batch_router.py new file mode 100644 index 0000000..f907dc0 --- /dev/null +++ b/th2_common/schema/message/impl/rabbitmq/byte/abstract_rabbit_byte_batch_router.py @@ -0,0 +1,34 @@ +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router_adapter import \ + RabbitMessageGroupBatchRouterAdapter +from th2_grpc_common.common_pb2 import AnyMessage, ByteMessageBatch, MessageGroup, MessageGroupBatch + + +class AbstractRabbitByteBatchRouter(RabbitMessageGroupBatchRouterAdapter): + + @staticmethod + def to_group_batch(message: ByteMessageBatch) -> MessageGroupBatch: + messages = [AnyMessage(byte_message=msg) for msg in message.messages] + group = MessageGroup(messages=messages) + + return MessageGroupBatch(groups=[group]) + + @staticmethod + def from_group_batch(message: MessageGroupBatch) -> ByteMessageBatch: + return ByteMessageBatch(messages=[ + anymsg.byte_message for group in message.groups for anymsg in group.messages if + anymsg.HasField('byte_message') + ]) diff --git a/th2_common/schema/message/impl/rabbitmq/byte/rabbit_cbor_batch_router.py b/th2_common/schema/message/impl/rabbitmq/byte/rabbit_cbor_batch_router.py new file mode 100644 index 0000000..bd27caf --- /dev/null +++ b/th2_common/schema/message/impl/rabbitmq/byte/rabbit_cbor_batch_router.py @@ -0,0 +1,29 @@ +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Set + +from th2_common.schema.message.impl.rabbitmq.byte.abstract_rabbit_byte_batch_router import AbstractRabbitByteBatchRouter +from th2_common.schema.message.queue_attribute import QueueAttribute + + +class RabbitCborBatchRouter(AbstractRabbitByteBatchRouter): + + @property + def required_subscribe_attributes(self) -> Set[str]: + return {QueueAttribute.SUBSCRIBE, QueueAttribute.CBOR} + + @property + def required_send_attributes(self) -> Set[str]: + return {QueueAttribute.PUBLISH, QueueAttribute.CBOR} diff --git a/th2_common/schema/message/impl/rabbitmq/byte/rabbit_json_batch_router.py b/th2_common/schema/message/impl/rabbitmq/byte/rabbit_json_batch_router.py new file mode 100644 index 0000000..daa9a3e --- /dev/null +++ b/th2_common/schema/message/impl/rabbitmq/byte/rabbit_json_batch_router.py @@ -0,0 +1,29 @@ +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Set + +from th2_common.schema.message.impl.rabbitmq.byte.abstract_rabbit_byte_batch_router import AbstractRabbitByteBatchRouter +from th2_common.schema.message.queue_attribute import QueueAttribute + + +class RabbitJsonBatchRouter(AbstractRabbitByteBatchRouter): + + @property + def required_subscribe_attributes(self) -> Set[str]: + return {QueueAttribute.SUBSCRIBE, QueueAttribute.JSON} + + @property + def required_send_attributes(self) -> Set[str]: + return {QueueAttribute.PUBLISH, QueueAttribute.JSON} diff --git a/th2_common/schema/message/impl/rabbitmq/configuration/rabbitmq_configuration.py b/th2_common/schema/message/impl/rabbitmq/configuration/rabbitmq_configuration.py index 364bb07..d53331f 100644 --- a/th2_common/schema/message/impl/rabbitmq/configuration/rabbitmq_configuration.py +++ b/th2_common/schema/message/impl/rabbitmq/configuration/rabbitmq_configuration.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, Optional from th2_common.schema.configuration.abstract_configuration import AbstractConfiguration class RabbitMQConfiguration(AbstractConfiguration): - def __init__(self, host, vHost, port, username, password, exchangeName, - prefetch_count: int = 100, subscriberName=None, **kwargs) -> None: + def __init__(self, + host: str, + vHost: str, + port: int, + username: str, + password: str, + exchangeName: str, + prefetch_count: int = 100, + subscriberName: Optional[str] = None, + **kwargs: Any) -> None: self.host = host self.vhost = vHost self.port = port @@ -28,4 +37,5 @@ def __init__(self, host, vHost, port, username, password, exchangeName, self.subscriber_name = subscriberName self.exchange_name = exchangeName self.prefetch_count = prefetch_count + self.check_unexpected_args(kwargs) diff --git a/th2_common/schema/message/impl/rabbitmq/configuration/subscribe_target.py b/th2_common/schema/message/impl/rabbitmq/configuration/subscribe_target.py index 50d5832..5f8e606 100644 --- a/th2_common/schema/message/impl/rabbitmq/configuration/subscribe_target.py +++ b/th2_common/schema/message/impl/rabbitmq/configuration/subscribe_target.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,14 +19,14 @@ def __init__(self, queue: str, routing_key: str) -> None: self.__queue = queue self.__routing_key = routing_key - def set_queue(self, queue: str): + def set_queue(self, queue: str) -> None: self.__queue = queue - def set_routing_key(self, routing_key: str): + def set_routing_key(self, routing_key: str) -> None: self.__routing_key = routing_key - def get_queue(self): + def get_queue(self) -> str: return self.__queue - def get_routing_key(self): + def get_routing_key(self) -> str: return self.__routing_key diff --git a/th2_common/schema/message/impl/rabbitmq/connection/connection_manager.py b/th2_common/schema/message/impl/rabbitmq/connection/connection_manager.py index 2cf8069..b92e78f 100644 --- a/th2_common/schema/message/impl/rabbitmq/connection/connection_manager.py +++ b/th2_common/schema/message/impl/rabbitmq/connection/connection_manager.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,18 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import logging import asyncio -from threading import Thread +from asyncio import AbstractEventLoop from contextlib import suppress +import logging +from threading import Thread +from typing import Any, Dict -import uvloop import aio_pika - -from google.protobuf.pyext._message import SetAllowOversizeProtos - -from th2_common.schema.message.configuration.message_configuration import ConnectionManagerConfiguration +from google.protobuf.pyext._message import SetAllowOversizeProtos # type: ignore +from th2_common.schema.message.configuration.message_configuration import MqConnectionConfiguration from th2_common.schema.message.impl.rabbitmq.configuration.rabbitmq_configuration import RabbitMQConfiguration from th2_common.schema.message.impl.rabbitmq.connection.consumer import Consumer from th2_common.schema.message.impl.rabbitmq.connection.publisher import Publisher @@ -43,8 +41,9 @@ class ConnectionManager: parameters for Consumer """ - def __init__(self, configuration: RabbitMQConfiguration, - connection_manager_configuration: ConnectionManagerConfiguration) -> None: + def __init__(self, + configuration: RabbitMQConfiguration, + connection_manager_configuration: MqConnectionConfiguration) -> None: SetAllowOversizeProtos(connection_manager_configuration.message_recursion_limit > 100) @@ -62,7 +61,7 @@ def __init__(self, configuration: RabbitMQConfiguration, self.connection_parameters) self.publisher = Publisher(self.connection_parameters) - self._loop = uvloop.new_event_loop() + self._loop = asyncio.get_event_loop() self.publisher_consumer_thread = Thread(target=self._start_background_loop) self.publisher_consumer_thread.start() @@ -81,7 +80,7 @@ def _start_background_loop(self) -> None: self._loop.set_exception_handler(self._handle_exception) self._loop.run_forever() - def _handle_exception(self, loop, context) -> None: + def _handle_exception(self, loop: AbstractEventLoop, context: Dict[str, Any]) -> Any: """Custom exception handling""" if isinstance(context.get('exception'), aio_pika.exceptions.CONNECTION_EXCEPTIONS): @@ -105,9 +104,13 @@ def close(self) -> None: except Exception as e: logger.exception(f'Error while stopping Publisher: {e}') + self.__metrics.disable() + graceful_shutdown = asyncio.run_coroutine_threadsafe(self._cancel_pending_tasks(), self._loop) graceful_shutdown.result() + self.publisher_consumer_thread.join() + async def _cancel_pending_tasks(self) -> None: """Coroutine that ensures graceful shutdown of event loop""" @@ -116,4 +119,5 @@ async def _cancel_pending_tasks(self) -> None: task.cancel() with suppress(asyncio.exceptions.CancelledError): await task - self._loop.stop() + + self._loop.call_soon_threadsafe(self._loop.stop) diff --git a/th2_common/schema/message/impl/rabbitmq/connection/consumer.py b/th2_common/schema/message/impl/rabbitmq/connection/consumer.py index e2cbaf2..e37982d 100644 --- a/th2_common/schema/message/impl/rabbitmq/connection/consumer.py +++ b/th2_common/schema/message/impl/rabbitmq/connection/consumer.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ import datetime import logging import time -from typing import Dict, Optional, Union, Callable, Any, Tuple +from types import FunctionType +from typing import Any, Dict, Optional, Tuple import aio_pika -from aio_pika.robust_connection import RobustConnection from aio_pika.robust_channel import RobustChannel +from aio_pika.robust_connection import RobustConnection from aio_pika.robust_queue import RobustQueue -from aio_pika.message import IncomingMessage -from th2_common.schema.message.configuration.message_configuration import ConnectionManagerConfiguration +from th2_common.schema.message.configuration.message_configuration import MqConnectionConfiguration logger = logging.getLogger(__name__) @@ -43,13 +43,13 @@ class Consumer: DEFAULT_SUBSCRIBER_NAME = 'rabbitmq_subscriber' def __init__(self, - connection_manager_configuration: ConnectionManagerConfiguration, + connection_manager_configuration: MqConnectionConfiguration, connection_parameters: dict) -> None: - self._subscriber_name: str = connection_manager_configuration.subscriber_name - self._prefetch_count: str = connection_manager_configuration.prefetch_count - self._subscribers: Dict[str, Tuple[RobustQueue, Callable]] = dict() - self._connection_parameters: Dict[str, Union[str, int]] = connection_parameters + self._subscriber_name: Optional[str] = connection_manager_configuration.subscriber_name + self._prefetch_count: int = connection_manager_configuration.prefetch_count + self._subscribers: Dict[str, Tuple[RobustQueue, FunctionType]] = {} + self._connection_parameters: Dict[str, Any] = connection_parameters self._connection: Optional[RobustConnection] = None self._channel: Optional[RobustChannel] = None self.__consumer_tag_id: int = -1 @@ -58,22 +58,22 @@ async def connect(self) -> None: """Coroutine that creates connection, channel for publisher and sets QOS""" loop = asyncio.get_running_loop() - while not self._connection: + while self._connection is None: try: self._connection = await aio_pika.connect_robust(loop=loop, **self._connection_parameters) except Exception as e: - logger.error(f"Exception was raised while connecting Consumer: {e}") + logger.error(f'Exception was raised while connecting Consumer: {e}') time.sleep(Consumer.DELAY_FOR_RECONNECTION) logger.info('Connection for Consumer has been created') while not self._channel: try: self._channel = await self._connection.channel() - await self._channel.set_qos(prefetch_count=self._prefetch_count) + await self._channel.set_qos(prefetch_count=self._prefetch_count) # type: ignore except Exception as e: - logger.error(f"Exception was raised while creating channel for Consumer: {e}") + logger.error(f'Exception was raised while creating channel for Consumer: {e}') time.sleep(Consumer.DELAY_FOR_RECONNECTION) - logger.info(f"Channel for Consumer has been created. QOS set to: {self._prefetch_count}") + logger.info(f'Channel for Consumer has been created. QOS set to: {self._prefetch_count}') def next_id(self) -> int: """Unique id for consumer_tag""" @@ -81,7 +81,7 @@ def next_id(self) -> int: self.__consumer_tag_id += 1 return self.__consumer_tag_id - def add_subscriber(self, queue_name: str, on_message_callback: Callable[[IncomingMessage], Any]) -> str: + def add_subscriber(self, queue_name: str, on_message_callback: FunctionType) -> str: """ Adding subscriber :param str queue_name: Name of the queue from where messages will be consumed @@ -98,19 +98,19 @@ def add_subscriber(self, queue_name: str, on_message_callback: Callable[[Incomin consumer_tag = f'{self._subscriber_name}.{self.next_id()}.{datetime.datetime.now()}' queue = asyncio.run_coroutine_threadsafe(self._get_queue_coroutine(queue_name), - self._connection.loop).result() + self._connection.loop).result() # type: ignore self._subscribers[consumer_tag] = (queue, on_message_callback) asyncio.run_coroutine_threadsafe(self._start_consuming(consumer_tag), - self._connection.loop) + self._connection.loop) # type: ignore return consumer_tag async def _get_queue_coroutine(self, queue_name: str) -> RobustQueue: """Coroutine that returns robust queue""" - return await self._channel.get_queue(name=queue_name) + return await self._channel.get_queue(name=queue_name) # type: ignore async def _start_consuming(self, consumer_tag: str) -> None: """Coroutine for consuming messages from queue""" @@ -121,7 +121,8 @@ async def _start_consuming(self, consumer_tag: str) -> None: def remove_subscriber(self, consumer_tag: str) -> None: """Remove subscriber and cancel consuming from queue""" - remove_consumer = asyncio.run_coroutine_threadsafe(self._stop_consuming(consumer_tag), self._connection.loop) + remove_consumer = asyncio.run_coroutine_threadsafe(self._stop_consuming(consumer_tag), + self._connection.loop) # type: ignore remove_consumer.result() self._subscribers.pop(consumer_tag) @@ -139,4 +140,4 @@ async def stop(self) -> None: for consumer_tag in self._subscribers: await self._stop_consuming(consumer_tag) - await self._connection.close() + await self._connection.close() # type: ignore diff --git a/th2_common/schema/message/impl/rabbitmq/connection/publisher.py b/th2_common/schema/message/impl/rabbitmq/connection/publisher.py index a463037..8fb3ec7 100644 --- a/th2_common/schema/message/impl/rabbitmq/connection/publisher.py +++ b/th2_common/schema/message/impl/rabbitmq/connection/publisher.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ import asyncio -import threading import logging +import threading import time -from typing import List, Union, Optional, Dict, Tuple +from typing import Any, Dict, List, Optional, Tuple import aio_pika from aio_pika import Message -from aio_pika.robust_connection import RobustConnection from aio_pika.robust_channel import RobustChannel +from aio_pika.robust_connection import RobustConnection from aio_pika.robust_exchange import RobustExchange @@ -48,8 +48,8 @@ class Publisher: DELAY_FOR_RECONNECTION = 5 PUBLISHING_COROUTINE_NAME = '_publish_message' - def __init__(self, connection_parameters: dict) -> None: - self._connection_parameters: Dict[str, Union[str, int]] = connection_parameters + def __init__(self, connection_parameters: Dict[str, Any]) -> None: + self._connection_parameters: Dict[str, Any] = connection_parameters self._connection: Optional[RobustConnection] = None self._channel: Optional[RobustChannel] = None self._exchange: Optional[RobustExchange] = None @@ -72,15 +72,15 @@ async def connect(self) -> None: except Exception as e: logger.error(f'Exception was raised while connecting Publisher: {e}') time.sleep(Publisher.DELAY_FOR_RECONNECTION) - logger.info("Connection for Publisher has been created") + logger.info('Connection for Publisher has been created') while not self._channel: try: self._channel = await self._connection.channel() except Exception as e: - logger.error(f"Exception was raised while creating channel for Publisher {e}") + logger.error(f'Exception was raised while creating channel for Publisher {e}') time.sleep(Publisher.DELAY_FOR_RECONNECTION) - logger.info("Channel for Publisher has been created") + logger.info('Channel for Publisher has been created') self._connection_event.set() self._publish_event.set() @@ -88,7 +88,7 @@ async def connect(self) -> None: async def _get_exchange_coroutine(self, name: str) -> RobustExchange: """Coroutine for getting exchange""" - return await self._channel.get_exchange(name=name) + return await self._channel.get_exchange(name=name) # type: ignore def _get_exchange(self, exchange_name: str) -> RobustExchange: """Returns an exchange object""" @@ -98,7 +98,7 @@ def _get_exchange(self, exchange_name: str) -> RobustExchange: while not exchange: try: exchange_future = asyncio.run_coroutine_threadsafe(self._get_exchange_coroutine(exchange_name), - self._connection.loop) + self._connection.loop) # type: ignore exchange = exchange_future.result() self._exchange_dict[exchange_name] = exchange except Exception as e: @@ -122,7 +122,7 @@ def publish_message(self, exchange = self._get_exchange(exchange_name) asyncio.run_coroutine_threadsafe(self._publish_message(exchange, routing_key, message), - self._connection.loop) + self._connection.loop) # type: ignore def _message_number_update(self) -> int: """Updates message number""" @@ -133,11 +133,11 @@ def _message_number_update(self) -> int: async def _publish_message(self, exchange: RobustExchange, routing_key: str, - message: bytes) -> None: + data: bytes) -> None: """Coroutine for publishing messages""" message_number = self._message_number_update() - message = Message(message) + message: Message = Message(data) try: await exchange.publish(message=message, routing_key=routing_key) @@ -145,8 +145,8 @@ async def _publish_message(self, self._connection_event.clear() self._publish_event.clear() if e.__class__.__name__ not in self._connection_exceptions: - logger.error(f"Connection issue: {e}. " - f"DELIVERY OF ALL ALREADY SENT MESSAGES IS NOT GUARANTEED") + logger.error(f'Connection issue: {e}. ' + f'DELIVERY OF ALL ALREADY SENT MESSAGES IS NOT GUARANTEED') self._connection_exceptions.append(e.__class__.__name__) failed = FailedMessage(exchange.name, routing_key, message.body, message_number) self._not_sent.append(failed) @@ -161,9 +161,9 @@ async def _wait_for_connection(self) -> None: while not self._connection_event.is_set(): await asyncio.sleep(Publisher.DELAY_FOR_RECONNECTION) - if self._connection.connected.is_set(): + if self._connection.connected.is_set(): # type: ignore self._connection_event.set() - logger.info("Connection was restored") + logger.info('Connection was restored') async def _republish_messages(self) -> None: """Republish messages that were failed due to connection issues""" @@ -195,9 +195,10 @@ async def stop(self) -> None: await self._wait_for_connection() publishing_tasks = [ - task for task in asyncio.all_tasks() if task.get_coro().__name__ == Publisher.PUBLISHING_COROUTINE_NAME + task for task in asyncio.all_tasks() + if task.get_coro().__name__ == Publisher.PUBLISHING_COROUTINE_NAME # type: ignore ] await asyncio.wait_for(asyncio.gather(*publishing_tasks), timeout=None) - await self._connection.close() + await self._connection.close() # type: ignore diff --git a/th2_common/schema/message/impl/rabbitmq/group/__init__.py b/th2_common/schema/message/impl/rabbitmq/group/__init__.py index e69de29..0449174 100644 --- a/th2_common/schema/message/impl/rabbitmq/group/__init__.py +++ b/th2_common/schema/message/impl/rabbitmq/group/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router.py b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router.py index c0b39b5..16aef17 100644 --- a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router.py +++ b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router.py @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from prometheus_client import Counter -from th2_grpc_common.common_pb2 import MessageGroupBatch +from typing import Set + +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer +from prometheus_client import Counter from th2_common.schema.message.configuration.message_configuration import QueueConfiguration from th2_common.schema.message.impl.rabbitmq.abstract_rabbit_message_router import AbstractRabbitMessageRouter from th2_common.schema.message.impl.rabbitmq.configuration.subscribe_target import SubscribeTarget @@ -27,6 +29,7 @@ from th2_common.schema.message.queue_attribute import QueueAttribute import th2_common.schema.metrics.common_metrics as common_metrics from th2_common.schema.metrics.metric_utils import update_dropped_metrics as dropped_metrics_updater +from th2_grpc_common.common_pb2 import MessageGroup, MessageGroupBatch class RabbitMessageGroupBatchRouter(AbstractRabbitMessageRouter): @@ -37,37 +40,41 @@ class RabbitMessageGroupBatchRouter(AbstractRabbitMessageRouter): 'Quantity of message groups dropped on sending', common_metrics.DEFAULT_LABELS) - def update_dropped_metrics(self, batch, pins): + def update_dropped_metrics(self, batch: MessageGroupBatch, *pins: str) -> None: for pin in pins: dropped_metrics_updater(batch, pin, self.OUTGOING_MSG_DROPPED, self.OUTGOING_MSG_GROUP_DROPPED) @property - def required_subscribe_attributes(self): - return {QueueAttribute.SUBSCRIBE.value} + def required_subscribe_attributes(self) -> Set[str]: + return {QueueAttribute.SUBSCRIBE} @property - def required_send_attributes(self): - return {QueueAttribute.PUBLISH.value} + def required_send_attributes(self) -> Set[str]: + return {QueueAttribute.PUBLISH} - def _get_messages(self, batch): + def _get_messages(self, batch: MessageGroupBatch) -> RepeatedCompositeFieldContainer: return batch.groups - def _create_batch(self): + def _create_batch(self) -> MessageGroupBatch: return MessageGroupBatch() - def _add_message(self, batch: MessageGroupBatch, group): + def _add_message(self, batch: MessageGroupBatch, group: MessageGroup) -> None: batch.groups.append(group) - def create_sender(self, connection_manager: ConnectionManager, - queue_configuration: QueueConfiguration, th2_pin) -> MessageSender: + def create_sender(self, + connection_manager: ConnectionManager, + queue_configuration: QueueConfiguration, + th2_pin: str) -> MessageSender: return RabbitMessageGroupBatchSender(connection_manager, queue_configuration.exchange, queue_configuration.routing_key, th2_pin=th2_pin) - def create_subscriber(self, connection_manager: ConnectionManager, - queue_configuration: QueueConfiguration, th2_pin) -> MessageSubscriber: + def create_subscriber(self, + connection_manager: ConnectionManager, + queue_configuration: QueueConfiguration, + th2_pin: str) -> MessageSubscriber: subscribe_target = SubscribeTarget(queue_configuration.queue, queue_configuration.routing_key) - return RabbitMessageGroupBatchSubscriber(connection_manager, - queue_configuration, - self.filter_strategy, - subscribe_target, + return RabbitMessageGroupBatchSubscriber(connection_manager=connection_manager, + subscribe_target=subscribe_target, + queue_configuration=queue_configuration, + filter_strategy=self.filter_strategy, # type: ignore th2_pin=th2_pin) diff --git a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router_adapter.py b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router_adapter.py index 99f047e..16ceb53 100644 --- a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router_adapter.py +++ b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_router_adapter.py @@ -12,39 +12,46 @@ # See the License for the specific language governing permissions and # limitations under the License. -from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router import \ - RabbitMessageGroupBatchRouter from abc import ABC, abstractmethod +from typing import Any, Callable +import google.protobuf.message +from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router import \ + RabbitMessageGroupBatchRouter from th2_common.schema.message.message_listener import MessageListener from th2_common.schema.message.subscriber_monitor import SubscriberMonitor +from th2_grpc_common.common_pb2 import MessageGroupBatch class RabbitMessageGroupBatchRouterAdapter(RabbitMessageGroupBatchRouter, ABC): - def send(self, message, *queue_attr): + def send(self, message: MessageGroupBatch, *queue_attr: str) -> None: super().send(self.to_group_batch(message), *queue_attr) - def send_all(self, message, *queue_attr): + def send_all(self, message: MessageGroupBatch, *queue_attr: str) -> None: super().send_all(self.to_group_batch(message), *queue_attr) - def subscribe(self, callback: MessageListener, *queue_attr) -> SubscriberMonitor: + def subscribe(self, callback: MessageListener, *queue_attr: str) -> SubscriberMonitor: return super().subscribe(self.get_converter(callback), *queue_attr) - def subscribe_all(self, callback: MessageListener, *queue_attr) -> SubscriberMonitor: + def subscribe_all(self, callback: MessageListener, *queue_attr: str) -> SubscriberMonitor: return super().subscribe_all(self.get_converter(callback), *queue_attr) @staticmethod @abstractmethod - def to_group_batch(message): + def to_group_batch(message: Any) -> MessageGroupBatch: pass @staticmethod @abstractmethod - def from_group_batch(message): + def from_group_batch(message: MessageGroupBatch) -> google.protobuf.message.Message: pass - def get_converter(self, callback: MessageListener): - old_handler = callback.handler - callback.handler = lambda attributes, message: old_handler(attributes, self.from_group_batch(message)) + def get_converter(self, callback: MessageListener) -> MessageListener: + old_handler: Callable = callback.handler + setattr( + callback, + 'handler', + lambda attributes, message: old_handler(attributes, self.from_group_batch(message)) + ) return callback diff --git a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_sender.py b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_sender.py index 6a668ad..b65fd14 100644 --- a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_sender.py +++ b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_sender.py @@ -14,18 +14,17 @@ from google.protobuf.json_format import MessageToJson from prometheus_client import Counter, Gauge -from th2_grpc_common.common_pb2 import MessageGroupBatch - from th2_common.schema.message.impl.rabbitmq.abstract_rabbit_sender import AbstractRabbitSender import th2_common.schema.metrics.common_metrics as common_metrics from th2_common.schema.metrics.metric_utils import update_total_metrics -from th2_common.schema.util.util import get_debug_string_group, get_session_alias_and_direction_group, get_sequence +from th2_common.schema.util.util import get_debug_string_group +from th2_grpc_common.common_pb2 import MessageGroupBatch class RabbitMessageGroupBatchSender(AbstractRabbitSender): OUTGOING_MSG_QUANTITY = Counter('th2_message_publish_total', 'Amount of individual messages sent', - common_metrics.DEFAULT_LABELS+(common_metrics.DEFAULT_MESSAGE_TYPE_LABEL_NAME, )) + common_metrics.DEFAULT_LABELS + (common_metrics.DEFAULT_MESSAGE_TYPE_LABEL_NAME, )) OUTGOING_MSG_GROUP_QUANTITY = Counter('th2_message_group_publish_total', 'Quantity of outgoing message groups', common_metrics.DEFAULT_LABELS) @@ -36,16 +35,19 @@ class RabbitMessageGroupBatchSender(AbstractRabbitSender): _TH2_TYPE = 'MESSAGE_GROUP' @staticmethod - def value_to_bytes(value: MessageGroupBatch): + def value_to_bytes(value: MessageGroupBatch) -> bytes: return value.SerializeToString() - def to_trace_string(self, value): + def to_trace_string(self, value: MessageGroupBatch) -> str: return MessageToJson(value) - def to_debug_string(self, value): + def to_debug_string(self, value: MessageGroupBatch) -> str: return get_debug_string_group(value) - def send(self, message): - update_total_metrics(message, self.th2_pin, self.OUTGOING_MSG_QUANTITY, self.OUTGOING_MSG_GROUP_QUANTITY, + def send(self, message: MessageGroupBatch) -> None: + update_total_metrics(message, + self.th2_pin, + self.OUTGOING_MSG_QUANTITY, + self.OUTGOING_MSG_GROUP_QUANTITY, self.OUTGOING_GROUP_SEQUENCE) super().send(message) diff --git a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_subscriber.py b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_subscriber.py index 493f188..0575fd7 100644 --- a/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_subscriber.py +++ b/th2_common/schema/message/impl/rabbitmq/group/rabbit_message_group_batch_subscriber.py @@ -12,58 +12,75 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer from google.protobuf.json_format import MessageToJson -from prometheus_client import Counter, Histogram, Gauge -from th2_grpc_common.common_pb2 import MessageGroupBatch - -import th2_common.schema.metrics.common_metrics as common_metrics +from prometheus_client import Counter, Gauge from th2_common.schema.message.impl.rabbitmq.abstract_rabbit_batch_subscriber import AbstractRabbitBatchSubscriber, \ Metadata -from th2_common.schema.util.util import get_debug_string_group, get_session_alias_and_direction_group, get_sequence +import th2_common.schema.metrics.common_metrics as common_metrics from th2_common.schema.metrics.metric_utils import update_dropped_metrics as util_dropped from th2_common.schema.metrics.metric_utils import update_total_metrics as util_total +from th2_common.schema.util.util import get_debug_string_group +from th2_grpc_common.common_pb2 import MessageGroupBatch class RabbitMessageGroupBatchSubscriber(AbstractRabbitBatchSubscriber): - INCOMING_MSG_QUANTITY = Counter('th2_message_subscribe_total', - 'Amount of received messages', - common_metrics.DEFAULT_LABELS+(common_metrics.DEFAULT_MESSAGE_TYPE_LABEL_NAME, )) - INCOMING_MSG_GROUP_QUANTITY = Counter('th2_message_group_subscribe_total', - 'Amount of received message groups', - common_metrics.DEFAULT_LABELS) - INCOMING_MSG_SEQUENCE = Gauge('th2_message_group_sequence_subscribe', - 'Last received sequence', - common_metrics.DEFAULT_LABELS) - INCOMING_MSG_DROPPED_QUANTITY = Counter('th2_message_dropped_subscribe_total', - 'Amount of messages dropped after filters', - common_metrics.DEFAULT_LABELS+(common_metrics.DEFAULT_MESSAGE_TYPE_LABEL_NAME, )) - INCOMING_MSG_GROUP_DROPPED_QUANTITY = Counter('th2_message_group_dropped_subscribe_total', - 'Amount of message groups dropped after filters', - common_metrics.DEFAULT_LABELS) + INCOMING_MSG_QUANTITY = Counter( + 'th2_message_subscribe_total', + 'Amount of received messages', + common_metrics.DEFAULT_LABELS + (common_metrics.DEFAULT_MESSAGE_TYPE_LABEL_NAME,) + ) + INCOMING_MSG_GROUP_QUANTITY = Counter( + 'th2_message_group_subscribe_total', + 'Amount of received message groups', + common_metrics.DEFAULT_LABELS + ) + INCOMING_MSG_SEQUENCE = Gauge( + 'th2_message_group_sequence_subscribe', + 'Last received sequence', + common_metrics.DEFAULT_LABELS + ) + INCOMING_MSG_DROPPED_QUANTITY = Counter( + 'th2_message_dropped_subscribe_total', + 'Amount of messages dropped after filters', + common_metrics.DEFAULT_LABELS + (common_metrics.DEFAULT_MESSAGE_TYPE_LABEL_NAME,) + ) + INCOMING_MSG_GROUP_DROPPED_QUANTITY = Counter( + 'th2_message_group_dropped_subscribe_total', + 'Amount of message groups dropped after filters', + common_metrics.DEFAULT_LABELS + ) _th2_type = 'MESSAGE_GROUP' - def update_dropped_metrics(self, batch): - util_dropped(batch, self.th2_pin, self.INCOMING_MSG_DROPPED_QUANTITY, self.INCOMING_MSG_GROUP_DROPPED_QUANTITY) + def update_dropped_metrics(self, batch: MessageGroupBatch) -> None: + util_dropped(batch, + self.th2_pin, + self.INCOMING_MSG_DROPPED_QUANTITY, + self.INCOMING_MSG_GROUP_DROPPED_QUANTITY) - def get_messages(self, batch) -> list: + def get_messages(self, batch: MessageGroupBatch) -> RepeatedCompositeFieldContainer: return batch.groups - def extract_metadata(self, message) -> Metadata: + def extract_metadata(self, message: MessageGroupBatch) -> Metadata: raise ValueError @staticmethod - def value_from_bytes(body): + def value_from_bytes(body: bytes) -> MessageGroupBatch: message_group_batch = MessageGroupBatch() message_group_batch.ParseFromString(body) return message_group_batch - def to_trace_string(self, value): + def to_trace_string(self, value: MessageGroupBatch) -> str: return MessageToJson(value) - def to_debug_string(self, value): + def to_debug_string(self, value: MessageGroupBatch) -> str: return get_debug_string_group(value) - def update_total_metrics(self, batch): - util_total(batch, self.th2_pin, self.INCOMING_MSG_QUANTITY, self.INCOMING_MSG_GROUP_QUANTITY, self.INCOMING_MSG_SEQUENCE) + def update_total_metrics(self, batch: MessageGroupBatch) -> None: + util_total(batch, + self.th2_pin, + self.INCOMING_MSG_QUANTITY, + self.INCOMING_MSG_GROUP_QUANTITY, + self.INCOMING_MSG_SEQUENCE) diff --git a/th2_common/schema/message/impl/rabbitmq/parsed/rabbit_parsed_batch_router.py b/th2_common/schema/message/impl/rabbitmq/parsed/rabbit_parsed_batch_router.py index aef2aee..7072046 100644 --- a/th2_common/schema/message/impl/rabbitmq/parsed/rabbit_parsed_batch_router.py +++ b/th2_common/schema/message/impl/rabbitmq/parsed/rabbit_parsed_batch_router.py @@ -12,32 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. - -from th2_grpc_common.common_pb2 import AnyMessage, MessageGroup, MessageGroupBatch, MessageBatch +from typing import Set from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router_adapter import \ RabbitMessageGroupBatchRouterAdapter from th2_common.schema.message.queue_attribute import QueueAttribute +from th2_grpc_common.common_pb2 import AnyMessage, MessageBatch, MessageGroup, MessageGroupBatch class RabbitParsedBatchRouter(RabbitMessageGroupBatchRouterAdapter): @property - def required_subscribe_attributes(self): - return {QueueAttribute.SUBSCRIBE.value, QueueAttribute.PARSED.value} + def required_subscribe_attributes(self) -> Set[str]: + return {QueueAttribute.SUBSCRIBE, QueueAttribute.PARSED} @property - def required_send_attributes(self): - return {QueueAttribute.PUBLISH.value, QueueAttribute.PARSED.value} + def required_send_attributes(self) -> Set[str]: + return {QueueAttribute.PUBLISH, QueueAttribute.PARSED} @staticmethod - def to_group_batch(message): + def to_group_batch(message: MessageBatch) -> MessageGroupBatch: messages = [AnyMessage(message=msg) for msg in message.messages] group = MessageGroup(messages=messages) - value = MessageGroupBatch(groups=[group]) - return value + + return MessageGroupBatch(groups=[group]) @staticmethod - def from_group_batch(message): - return MessageBatch(messages=[anymsg.message for group in message.groups for anymsg in group.messages if - anymsg.HasField('message')]) + def from_group_batch(message: MessageGroupBatch) -> MessageBatch: + return MessageBatch(messages=[ + anymsg.message for group in message.groups for anymsg in group.messages if + anymsg.HasField('message') + ]) diff --git a/th2_common/schema/message/impl/rabbitmq/raw/rabbit_raw_batch_router.py b/th2_common/schema/message/impl/rabbitmq/raw/rabbit_raw_batch_router.py index 2b6fb5e..08796e1 100644 --- a/th2_common/schema/message/impl/rabbitmq/raw/rabbit_raw_batch_router.py +++ b/th2_common/schema/message/impl/rabbitmq/raw/rabbit_raw_batch_router.py @@ -12,31 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -from th2_grpc_common.common_pb2 import AnyMessage, MessageGroup, MessageGroupBatch, RawMessageBatch +from typing import Set from th2_common.schema.message.impl.rabbitmq.group.rabbit_message_group_batch_router_adapter import \ RabbitMessageGroupBatchRouterAdapter from th2_common.schema.message.queue_attribute import QueueAttribute +from th2_grpc_common.common_pb2 import AnyMessage, MessageGroup, MessageGroupBatch, RawMessageBatch class RabbitRawBatchRouter(RabbitMessageGroupBatchRouterAdapter): @property - def required_subscribe_attributes(self): - return {QueueAttribute.SUBSCRIBE.value, QueueAttribute.RAW.value} + def required_subscribe_attributes(self) -> Set[str]: + return {QueueAttribute.SUBSCRIBE, QueueAttribute.RAW} @property - def required_send_attributes(self): - return {QueueAttribute.PUBLISH.value, QueueAttribute.RAW.value} + def required_send_attributes(self) -> Set[str]: + return {QueueAttribute.PUBLISH, QueueAttribute.RAW} @staticmethod - def to_group_batch(message): + def to_group_batch(message: RawMessageBatch) -> MessageGroupBatch: messages = [AnyMessage(raw_message=msg) for msg in message.messages] group = MessageGroup(messages=messages) - value = MessageGroupBatch(groups=[group]) - return value + + return MessageGroupBatch(groups=[group]) @staticmethod - def from_group_batch(message): - return RawMessageBatch(messages=[anymsg.raw_message for group in message.groups for anymsg in group.messages if - anymsg.HasField('raw_message')]) + def from_group_batch(message: MessageGroupBatch) -> RawMessageBatch: + return RawMessageBatch(messages=[ + anymsg.raw_message for group in message.groups for anymsg in group.messages if + anymsg.HasField('raw_message') + ]) diff --git a/th2_common/schema/message/message_listener.py b/th2_common/schema/message/message_listener.py index 011761f..c18178a 100644 --- a/th2_common/schema/message/message_listener.py +++ b/th2_common/schema/message/message_listener.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. - from abc import ABC, abstractmethod +from typing import Tuple + +from th2_grpc_common.common_pb2 import Message class MessageListener(ABC): @abstractmethod - def handler(self, attributes: tuple, message): + def handler(self, attributes: Tuple, message: Message) -> None: pass - def on_close(self): + def on_close(self) -> None: pass diff --git a/th2_common/schema/message/message_router.py b/th2_common/schema/message/message_router.py index e53c798..a33c8b4 100644 --- a/th2_common/schema/message/message_router.py +++ b/th2_common/schema/message/message_router.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ from abc import ABC, abstractmethod from threading import Lock +from typing import Any -from th2_common.schema.message.configuration.message_configuration import MessageRouterConfiguration, QueueConfiguration +from th2_common.schema.message.configuration.message_configuration import MessageRouterConfiguration from th2_common.schema.message.impl.rabbitmq.connection.connection_manager import ConnectionManager from th2_common.schema.message.message_listener import MessageListener from th2_common.schema.message.message_sender import MessageSender @@ -28,7 +29,8 @@ class MessageRouter(ABC): Interface for send and receive RabbitMQ messages """ - def __init__(self, connection_manager: ConnectionManager, + def __init__(self, + connection_manager: ConnectionManager, configuration: MessageRouterConfiguration) -> None: self.configuration = configuration self.connection_manager = connection_manager @@ -36,7 +38,7 @@ def __init__(self, connection_manager: ConnectionManager, self.sender_lock = Lock() @abstractmethod - def subscribe(self, callback: MessageListener, *queue_attr) -> SubscriberMonitor: + def subscribe(self, callback: MessageListener, *queue_attr: str) -> SubscriberMonitor: """ RabbitMQ queue by intersection schemas queues attributes :param callback: listener @@ -46,7 +48,7 @@ def subscribe(self, callback: MessageListener, *queue_attr) -> SubscriberMonitor pass @abstractmethod - def subscribe_all(self, callback: MessageListener, *queue_attr) -> SubscriberMonitor: + def subscribe_all(self, callback: MessageListener, *queue_attr: str) -> SubscriberMonitor: """ RabbitMQ queues :param callback: listener @@ -55,7 +57,7 @@ def subscribe_all(self, callback: MessageListener, *queue_attr) -> SubscriberMon """ @abstractmethod - def unsubscribe_all(self): + def unsubscribe_all(self) -> None: """ Unsubscribe from all queues :return: @@ -63,21 +65,25 @@ def unsubscribe_all(self): pass @abstractmethod - def send(self, message, *queue_attr): + def send(self, message: Any, *queue_attr: str) -> None: pass @abstractmethod - def send_all(self, message, *queue_attr): + def send_all(self, message: Any, *queue_attr: str) -> None: pass @abstractmethod - def get_subscriber(self, queue_alias) -> MessageSubscriber: + def get_subscriber(self, queue_alias: str) -> MessageSubscriber: pass @abstractmethod - def get_sender(self, queue_alias) -> MessageSender: + def get_sender(self, queue_alias: str) -> MessageSender: pass @abstractmethod - def close_connection(self, queue_alias): + def close_connection(self, queue_alias: str) -> None: + pass + + @abstractmethod + def close(self) -> None: pass diff --git a/th2_common/schema/message/message_sender.py b/th2_common/schema/message/message_sender.py index 84fba65..09ce566 100644 --- a/th2_common/schema/message/message_sender.py +++ b/th2_common/schema/message/message_sender.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,12 +13,13 @@ # limitations under the License. from abc import ABC, abstractmethod +from typing import Any class MessageSender(ABC): @abstractmethod - def start(self): + def start(self) -> None: pass @abstractmethod @@ -26,9 +27,9 @@ def is_close(self) -> bool: pass @abstractmethod - def send(self, message): + def send(self, message: Any) -> None: pass @abstractmethod - def close(self): + def close(self) -> None: pass diff --git a/th2_common/schema/message/message_subscriber.py b/th2_common/schema/message/message_subscriber.py index 5d46a39..a7ce9ed 100644 --- a/th2_common/schema/message/message_subscriber.py +++ b/th2_common/schema/message/message_subscriber.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ class MessageSubscriber(ABC): @abstractmethod - def start(self): + def start(self) -> None: pass @abstractmethod @@ -28,9 +28,9 @@ def is_close(self) -> bool: pass @abstractmethod - def add_listener(self, message_listener: MessageListener): + def add_listener(self, message_listener: MessageListener) -> None: pass @abstractmethod - def close(self): + def close(self) -> None: pass diff --git a/th2_common/schema/message/queue_attribute.py b/th2_common/schema/message/queue_attribute.py index 1d3c4bd..3fb893a 100644 --- a/th2_common/schema/message/queue_attribute.py +++ b/th2_common/schema/message/queue_attribute.py @@ -1,7 +1,21 @@ +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import enum -class QueueAttribute(enum.Enum): +class QueueAttribute(str, enum.Enum): FIRST = 'first' SECOND = 'second' SUBSCRIBE = 'subscribe' @@ -10,3 +24,5 @@ class QueueAttribute(enum.Enum): RAW = 'raw' EVENT = 'event' STORE = 'store' + CBOR = 'cbor' + JSON = 'json' diff --git a/th2_common/schema/message/subscriber_monitor.py b/th2_common/schema/message/subscriber_monitor.py index c2c5946..b9701c8 100644 --- a/th2_common/schema/message/subscriber_monitor.py +++ b/th2_common/schema/message/subscriber_monitor.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,5 +18,5 @@ class SubscriberMonitor(ABC): @abstractmethod - def unsubscribe(self): + def unsubscribe(self) -> None: pass diff --git a/th2_common/schema/metrics/abstract_metric.py b/th2_common/schema/metrics/abstract_metric.py index 885d4f1..a63bafa 100644 --- a/th2_common/schema/metrics/abstract_metric.py +++ b/th2_common/schema/metrics/abstract_metric.py @@ -1,4 +1,5 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2022 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,36 +14,28 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import Set from th2_common.schema.metrics.metric import Metric -from th2_common.schema.metrics.metric_monitor import MetricMonitor class AbstractMetric(Metric, ABC): - __disabled_monitors: Set[MetricMonitor] = set() + def __init__(self) -> None: + self._enabled: bool = False - @property - def enabled(self) -> bool: - return not self.__disabled_monitors + def is_enabled(self) -> bool: + return self._enabled - def create_monitor(self, name: str) -> MetricMonitor: - return MetricMonitor(name, self) + def enable(self) -> None: + if not self._enabled: + self._enabled = True + self.on_value_change(self._enabled) - def is_enabled(self, monitor: MetricMonitor) -> bool: - return monitor not in self.__disabled_monitors - - def enable(self, monitor: MetricMonitor): - if not self.is_enabled(monitor): - self.__disabled_monitors.remove(monitor) - self.on_value_change(self.enabled) - - def disable(self, monitor: MetricMonitor): - if self.is_enabled(monitor): - self.__disabled_monitors.add(monitor) - self.on_value_change(False) + def disable(self) -> None: + if self._enabled: + self._enabled = False + self.on_value_change(self._enabled) @abstractmethod - def on_value_change(self, value: bool): + def on_value_change(self, value: bool) -> None: pass diff --git a/th2_common/schema/metrics/aggregating_metric.py b/th2_common/schema/metrics/aggregating_metric.py index 621de66..dae0637 100644 --- a/th2_common/schema/metrics/aggregating_metric.py +++ b/th2_common/schema/metrics/aggregating_metric.py @@ -1,4 +1,4 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,8 @@ from typing import List +from th2_common.schema.metrics.aggregating_metric_monitor import AggregatingMetricMonitor from th2_common.schema.metrics.metric import Metric -from th2_common.schema.metrics.metric_monitor import MetricMonitor class AggregatingMetric(Metric): @@ -23,20 +23,16 @@ class AggregatingMetric(Metric): def __init__(self, metrics: List[Metric]) -> None: self.metrics = metrics - @property - def enabled(self) -> bool: - return all(metric.enabled for metric in self.metrics) + def create_monitor(self, name: str) -> AggregatingMetricMonitor: + return AggregatingMetricMonitor(name, self) - def create_monitor(self, name: str) -> MetricMonitor: - return MetricMonitor(name, self) + def is_enabled(self) -> bool: + return all(metric.is_enabled() for metric in self.metrics) - def is_enabled(self, monitor: MetricMonitor) -> bool: - return all(metric.is_enabled(monitor) for metric in self.metrics) - - def enable(self, monitor: MetricMonitor): + def enable(self) -> None: for metric in self.metrics: - metric.enable(monitor) + metric.enable() - def disable(self, monitor: MetricMonitor): + def disable(self) -> None: for metric in self.metrics: - metric.disable(monitor) + metric.disable() diff --git a/th2_common/schema/strategy/field_extraction/impl/th2_batch_msg_field_extraction.py b/th2_common/schema/metrics/aggregating_metric_monitor.py similarity index 51% rename from th2_common/schema/strategy/field_extraction/impl/th2_batch_msg_field_extraction.py rename to th2_common/schema/metrics/aggregating_metric_monitor.py index 9ac885b..b5cfba1 100644 --- a/th2_common/schema/strategy/field_extraction/impl/th2_batch_msg_field_extraction.py +++ b/th2_common/schema/metrics/aggregating_metric_monitor.py @@ -1,4 +1,4 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from th2_common.schema.metrics.metric import Metric -import th2_grpc_common.common_pb2 -from google.protobuf.message import Message -from th2_common.schema.strategy.field_extraction.abstract_th2_msg_field_extraction import AbstractTh2MsgFieldExtraction +class AggregatingMetricMonitor(Metric): + def __init__(self, name: str, aggregating_metric: Metric) -> None: + self.name = name + self.aggregating_metric = aggregating_metric -class Th2BatchMsgFieldExtraction(AbstractTh2MsgFieldExtraction): + def is_enabled(self) -> bool: + return self.aggregating_metric.is_enabled() - def parse_message(self, message: Message) -> th2_grpc_common.common_pb2.Message: - return message + def enable(self) -> None: + self.aggregating_metric.enable() + + def disable(self) -> None: + self.aggregating_metric.disable() diff --git a/th2_common/schema/metrics/common_metrics.py b/th2_common/schema/metrics/common_metrics.py index 556c74d..cc817ad 100644 --- a/th2_common/schema/metrics/common_metrics.py +++ b/th2_common/schema/metrics/common_metrics.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,35 +15,49 @@ from typing import Tuple from th2_common.schema.metrics.aggregating_metric import AggregatingMetric +from th2_common.schema.metrics.aggregating_metric_monitor import AggregatingMetricMonitor from th2_common.schema.metrics.file_metric import FileMetric from th2_common.schema.metrics.prometheus_metric import PrometheusMetric DEFAULT_BUCKETS = [0.000_25, 0.000_5, 0.001, 0.005, 0.010, 0.015, 0.025, 0.050, 0.100, 0.250, 0.500, 1.0] -DEFAULT_SESSION_ALIAS_LABEL_NAME: str = "session_alias" -DEFAULT_DIRECTION_LABEL_NAME: str = "direction" +DEFAULT_SESSION_ALIAS_LABEL_NAME: str = 'session_alias' +DEFAULT_DIRECTION_LABEL_NAME: str = 'direction' DEFAULT_EXCHANGE_LABEL_NAME: str = 'exchange' DEFAULT_ROUTING_KEY_LABEL_NAME: str = 'routing_key' DEFAULT_QUEUE_LABEL_NAME: str = 'queue' DEFAULT_MESSAGE_TYPE_LABEL_NAME: str = 'message_type' DEFAULT_TH2_PIN_LABEL_NAME: str = 'th2_pin' DEFAULT_TH2_TYPE_LABEL_NAME: str = 'th2_type' -DEFAULT_LABELS: Tuple[str, str, str] = (DEFAULT_TH2_PIN_LABEL_NAME, DEFAULT_SESSION_ALIAS_LABEL_NAME, DEFAULT_DIRECTION_LABEL_NAME) +DEFAULT_LABELS: Tuple[str, str, str] = ( + DEFAULT_TH2_PIN_LABEL_NAME, + DEFAULT_SESSION_ALIAS_LABEL_NAME, + DEFAULT_DIRECTION_LABEL_NAME +) EMPTY_LABELS: Tuple[str, str] = ('', '') UNKNOWN_LABELS: Tuple[str, str] = ('unknown', 'unknown') -SENDER_LABELS = DEFAULT_TH2_PIN_LABEL_NAME, DEFAULT_TH2_TYPE_LABEL_NAME, DEFAULT_EXCHANGE_LABEL_NAME, DEFAULT_ROUTING_KEY_LABEL_NAME -SUBSCRIBER_LABELS = DEFAULT_TH2_PIN_LABEL_NAME, DEFAULT_TH2_TYPE_LABEL_NAME, DEFAULT_QUEUE_LABEL_NAME -TH2_MESSAGE_TYPES: dict = {'raw': 'RAW_MESSAGE', 'parsed': 'MESSAGE'} - -LIVENESS_ARBITER = AggregatingMetric([PrometheusMetric("th2_liveness", "Service liveness"), FileMetric('healthy')]) -READINESS_ARBITER = AggregatingMetric([PrometheusMetric("th2_readiness", "Service readiness"), FileMetric('ready')]) - - -def register_liveness(name: str): +SENDER_LABELS = ( + DEFAULT_TH2_PIN_LABEL_NAME, + DEFAULT_TH2_TYPE_LABEL_NAME, + DEFAULT_EXCHANGE_LABEL_NAME, + DEFAULT_ROUTING_KEY_LABEL_NAME +) +SUBSCRIBER_LABELS = ( + DEFAULT_TH2_PIN_LABEL_NAME, + DEFAULT_TH2_TYPE_LABEL_NAME, + DEFAULT_QUEUE_LABEL_NAME +) +TH2_MESSAGE_TYPES: dict = {'byte': 'BYTE_MESSAGE', 'raw': 'RAW_MESSAGE', 'parsed': 'MESSAGE'} + +LIVENESS_ARBITER = AggregatingMetric([PrometheusMetric('th2_liveness', 'Service liveness'), FileMetric('healthy')]) +READINESS_ARBITER = AggregatingMetric([PrometheusMetric('th2_readiness', 'Service readiness'), FileMetric('ready')]) + + +def register_liveness(name: str) -> AggregatingMetricMonitor: return LIVENESS_ARBITER.create_monitor(name) -def register_readiness(name: str): +def register_readiness(name: str) -> AggregatingMetricMonitor: return READINESS_ARBITER.create_monitor(name) @@ -57,10 +71,10 @@ def __init__(self, obj: object) -> None: self.liveness_monitor = register_liveness(f'{obj.__class__.__name__}_liveness_{hash(obj)}') self.readiness_monitor = register_readiness(f'{obj.__class__.__name__}_readiness_{hash(obj)}') - def enable(self): + def enable(self) -> None: self.liveness_monitor.enable() self.readiness_monitor.enable() - def disable(self): + def disable(self) -> None: self.liveness_monitor.disable() self.readiness_monitor.disable() diff --git a/th2_common/schema/metrics/file_metric.py b/th2_common/schema/metrics/file_metric.py index ff3ec9c..bcbb602 100644 --- a/th2_common/schema/metrics/file_metric.py +++ b/th2_common/schema/metrics/file_metric.py @@ -1,4 +1,4 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import tempfile from pathlib import Path +import tempfile from th2_common.schema.metrics.abstract_metric import AbstractMetric @@ -25,7 +25,9 @@ def __init__(self, filename: str) -> None: if self.filename.exists(): self.filename.unlink() - def on_value_change(self, value: bool): + super().__init__() + + def on_value_change(self, value: bool) -> None: if value: try: self.filename.touch() diff --git a/th2_common/schema/metrics/metric.py b/th2_common/schema/metrics/metric.py index 39ad579..df8702f 100644 --- a/th2_common/schema/metrics/metric.py +++ b/th2_common/schema/metrics/metric.py @@ -1,4 +1,5 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2022 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,45 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABC -import th2_common.schema.metrics.metric_monitor as metric_monitor +class Metric: - -class Metric(ABC): - - enabled: bool = None - - def create_monitor(self, name: str) -> 'metric_monitor.MetricMonitor': - """ - Creates new monitor for the metric - - :param name: monitor name - :return: MetricMonitor which can change metrics value - """ - pass - - def is_enabled(self, monitor: 'metric_monitor.MetricMonitor') -> bool: + def is_enabled(self) -> bool: """ Checks if status of a monitor is `enabled` - :param monitor: MetricMonitor :return: Status (bool) of MetricMonitor """ pass - def enable(self, monitor: 'metric_monitor.MetricMonitor'): + def enable(self) -> None: """ Changes status of a monitor to `enabled` - :param monitor: MetricMonitor """ pass - def disable(self, monitor: 'metric_monitor.MetricMonitor'): + def disable(self) -> None: """ Changes status of a monitor to `disabled` - :param monitor: MetricMonitor """ pass diff --git a/th2_common/schema/metrics/metric_monitor.py b/th2_common/schema/metrics/metric_monitor.py deleted file mode 100644 index f0648b8..0000000 --- a/th2_common/schema/metrics/metric_monitor.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import th2_common.schema.metrics.metric as metric - - -class MetricMonitor: - - def __init__(self, name: str, arbiter: 'metric.Metric') -> None: - self.name = name - self.arbiter = arbiter - - @property - def is_enabled(self) -> bool: - return self.arbiter.is_enabled(self) - - @is_enabled.setter - def is_enabled(self, value): - if value: - self.enable() - else: - self.disable() - - @property - def is_metric_enabled(self): - return self.arbiter.enabled - - def enable(self): - self.arbiter.enable(self) - - def disable(self): - self.arbiter.disable(self) diff --git a/th2_common/schema/metrics/metric_utils.py b/th2_common/schema/metrics/metric_utils.py index 13d3a46..79d171d 100644 --- a/th2_common/schema/metrics/metric_utils.py +++ b/th2_common/schema/metrics/metric_utils.py @@ -12,31 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer +from prometheus_client import Counter, Gauge from th2_common.schema.metrics import common_metrics -from th2_common.schema.util.util import get_session_alias_and_direction_group, get_sequence +from th2_common.schema.util.util import get_sequence, get_session_alias_and_direction_group +from th2_grpc_common.common_pb2 import MessageGroupBatch -def update_total_metrics(batch, th2_pin, message_counter, group_counter, sequence_gauge): +def update_total_metrics(batch: MessageGroupBatch, + th2_pin: str, + message_counter: Counter, + group_counter: Counter, + sequence_gauge: Gauge) -> None: for group in batch.groups: gr_labels = (th2_pin, ) + get_session_alias_and_direction_group(group.messages[0]) - update_message_metrics(group.messages, message_counter, gr_labels) + update_message_metrics(group.messages, message_counter, *gr_labels) if group_counter: group_counter.labels(*gr_labels).inc() if sequence_gauge: - sequence_gauge.labels(*gr_labels).set(get_sequence(group)) + sequence_gauge.labels(*gr_labels).set(get_sequence(group)) # type: ignore -def update_message_metrics(messages, counter, labels): +def update_message_metrics(messages: RepeatedCompositeFieldContainer, counter: Counter, *labels: str) -> None: for msg in messages: if msg.HasField('raw_message'): counter.labels(*labels, common_metrics.TH2_MESSAGE_TYPES['raw']).inc() + elif msg.HasField('byte_message'): + counter.labels(*labels, common_metrics.TH2_MESSAGE_TYPES['byte']).inc() elif msg.HasField('message'): counter.labels(*labels, common_metrics.TH2_MESSAGE_TYPES['parsed']).inc() -def update_dropped_metrics(batch, th2_pin, message_counter, group_counter): +def update_dropped_metrics(batch: MessageGroupBatch, + th2_pin: str, + message_counter: Counter, + group_counter: Counter) -> None: for group in batch.groups: labels = (th2_pin, ) + get_session_alias_and_direction_group(group.messages[0]) - update_message_metrics(group.messages, message_counter, labels) + update_message_metrics(group.messages, message_counter, *labels) if group_counter: group_counter.labels(*labels).inc() diff --git a/th2_common/schema/metrics/prometheus_configuration.py b/th2_common/schema/metrics/prometheus_configuration.py index 76eb31e..e14c7c9 100644 --- a/th2_common/schema/metrics/prometheus_configuration.py +++ b/th2_common/schema/metrics/prometheus_configuration.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any + from th2_common.schema.configuration.abstract_configuration import AbstractConfiguration class PrometheusConfiguration(AbstractConfiguration): - def __init__(self, host="0.0.0.0", port=9752, enabled=False, **kwargs) -> None: + def __init__(self, host: str = '0.0.0.0', port: int = 9752, enabled: bool = False, **kwargs: Any) -> None: self.host = host self.port = port self.enabled = enabled diff --git a/th2_common/schema/metrics/prometheus_metric.py b/th2_common/schema/metrics/prometheus_metric.py index 9184754..416157c 100644 --- a/th2_common/schema/metrics/prometheus_metric.py +++ b/th2_common/schema/metrics/prometheus_metric.py @@ -1,4 +1,4 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ # limitations under the License. from prometheus_client import Gauge - from th2_common.schema.metrics.abstract_metric import AbstractMetric @@ -22,5 +21,7 @@ class PrometheusMetric(AbstractMetric): def __init__(self, name: str, documentation: str) -> None: self.metric = Gauge(name=name, documentation=documentation) - def on_value_change(self, value: bool): + super().__init__() + + def on_value_change(self, value: bool) -> None: self.metric.set(1.0 if value else 0.0) diff --git a/th2_common/schema/metrics/prometheus_server.py b/th2_common/schema/metrics/prometheus_server.py index ac56a3e..ca895cc 100644 --- a/th2_common/schema/metrics/prometheus_server.py +++ b/th2_common/schema/metrics/prometheus_server.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,19 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -from prometheus_client.exposition import start_wsgi_server +from threading import Thread +from typing import Optional +from wsgiref.simple_server import make_server, WSGIServer + +from prometheus_client import make_wsgi_app +from prometheus_client.exposition import _SilentHandler class PrometheusServer: - def __init__(self, port=8000, host='localhost'): - self.stopped = None + def __init__(self, port: int = 8000, host: str = 'localhost') -> None: + self.stopped: Optional[bool] = None self.port = port self.host = host + self.httpd: Optional[WSGIServer] = None + self.server_thread: Optional[Thread] = None - def run(self): - start_wsgi_server(self.port, self.host) - self.stopped = False + def run(self) -> None: + if self.httpd is None or self.stopped is True: + self.stopped = False + app = make_wsgi_app() + self.httpd = make_server(self.host, self.port, app, handler_class=_SilentHandler) + self.server_thread = Thread(target=self.httpd.serve_forever, daemon=True) + self.server_thread.start() - def stop(self): - self.stopped = True + def stop(self) -> None: + if self.stopped is False and self.httpd is not None and self.server_thread is not None: + self.httpd.shutdown() + self.httpd.server_close() + self.server_thread.join() + self.stopped = True diff --git a/th2_common/schema/strategy/field_extraction/abstract_th2_msg_field_extraction.py b/th2_common/schema/strategy/field_extraction/abstract_th2_msg_field_extraction.py deleted file mode 100644 index 8865e13..0000000 --- a/th2_common/schema/strategy/field_extraction/abstract_th2_msg_field_extraction.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from abc import ABC, abstractmethod - -import google.protobuf.message -from th2_grpc_common.common_pb2 import Direction, Message - -from th2_common.schema.strategy.field_extraction.field_extraction_strategy import FieldExtractionStrategy - - -class AbstractTh2MsgFieldExtraction(FieldExtractionStrategy, ABC): - - SESSION_ALIAS_KEY = 'session_alias' - MESSAGE_TYPE_KEY = 'message_type' - DIRECTION_KEY = 'direction' - - def get_fields(self, message: google.protobuf.message.Message) -> {str: str}: - th2msg = self.parse_message(message) - message_id = th2msg.metadata.id - - message_fields = dict() - for field_name in th2msg.fields.keys(): - message_fields[field_name] = th2msg.fields[field_name].simple_value - metadata_msg_fields = {self.SESSION_ALIAS_KEY: message_id.connection_id.session_alias, - self.MESSAGE_TYPE_KEY: th2msg.metadata.message_type, - self.DIRECTION_KEY: Direction.Name(message_id.direction)} - message_fields.update(metadata_msg_fields) - return message_fields - - @abstractmethod - def parse_message(self, message: google.protobuf.message.Message) -> Message: - pass diff --git a/th2_common/schema/strategy/field_extraction/th2_msg_field_extraction.py b/th2_common/schema/strategy/field_extraction/th2_msg_field_extraction.py new file mode 100644 index 0000000..7da65d3 --- /dev/null +++ b/th2_common/schema/strategy/field_extraction/th2_msg_field_extraction.py @@ -0,0 +1,67 @@ +# Copyright 2022-2022 Exactpro (Exactpro Systems Limited) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Any, Dict + +from th2_common_utils import message_to_dict +from th2_grpc_common.common_pb2 import AnyMessage, Direction + + +logger = logging.getLogger(__name__) + + +class Th2MsgFieldExtraction: + + SESSION_ALIAS_KEY: str = 'session_alias' + MESSAGE_TYPE_KEY: str = 'message_type' + DIRECTION_KEY: str = 'direction' + PROTOCOL_KEY: str = 'protocol' + + def get_fields(self, any_message: AnyMessage) -> Dict[str, Any]: + if any_message.HasField('message'): + return self._message_with_metadata_to_dict(any_message) + + elif any_message.HasField('raw_message'): + return self._raw_message_metadata_to_dict(any_message) + + elif any_message.HasField('byte_message'): + return self._byte_message_metadata_to_dict(any_message) + + else: + logger.warning(f'Cannot get fields from {type(any_message)} object') + return {} + + def _message_with_metadata_to_dict(self, any_message: AnyMessage) -> Dict[str, Any]: + message_dict = message_to_dict(any_message.message) + + return {**message_dict['fields'], **message_dict['metadata']} + + def _raw_message_metadata_to_dict(self, any_message: AnyMessage) -> Dict[str, Any]: + raw_message_metadata = any_message.raw_message.metadata + + return { + self.SESSION_ALIAS_KEY: raw_message_metadata.id.connection_id.session_alias, + self.DIRECTION_KEY: Direction.Name(raw_message_metadata.id.direction), + self.PROTOCOL_KEY: raw_message_metadata.protocol + } + + def _byte_message_metadata_to_dict(self, any_message: AnyMessage) -> Dict[str, Any]: + byte_message_metadata = any_message.byte_message.metadata + + return { + self.SESSION_ALIAS_KEY: byte_message_metadata.id.connection_id.session_alias, + self.DIRECTION_KEY: Direction.Name(byte_message_metadata.id.direction), + self.PROTOCOL_KEY: byte_message_metadata.protocol + } diff --git a/th2_common/schema/strategy/route/impl/filter_routing_strategy.py b/th2_common/schema/strategy/route/impl/filter_routing_strategy.py deleted file mode 100644 index 9d563c2..0000000 --- a/th2_common/schema/strategy/route/impl/filter_routing_strategy.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from google.protobuf.message import Message - -from th2_common.schema.filter.strategy.impl.default_filter_strategy import DefaultFilterStrategy -from th2_common.schema.grpc.configuration.grpc_configuration import GrpcRawFilterStrategy -from th2_common.schema.strategy.route.routing_strategy import RoutingStrategy - - -class FilterRoutingStrategy(RoutingStrategy): - - def __init__(self, configuration: GrpcRawFilterStrategy) -> None: - self.__filter_strategy = DefaultFilterStrategy() - self.__grpc_configuration = configuration - - def get_endpoint(self, message: Message): - endpoint: set = self.__filter(message) - if len(endpoint) != 1: - raise Exception('Wrong size of endpoints for send. Should be equal to 1') - return endpoint.__iter__().__next__() - - def __filter(self, message: Message) -> {str}: - endpoints = set() - for fields_filter in self.__grpc_configuration.filters: - if len(fields_filter.message) == 0 and len(fields_filter.metadata) == 0 \ - or self.__filter_strategy.verify(message=message, router_filter=fields_filter): - endpoints.add(fields_filter.endpoint) - return endpoints diff --git a/th2_common/schema/strategy/route/impl/robin_routing_strategy.py b/th2_common/schema/strategy/route/impl/robin_routing_strategy.py deleted file mode 100644 index d1bb00e..0000000 --- a/th2_common/schema/strategy/route/impl/robin_routing_strategy.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2020-2020 Exactpro (Exactpro Systems Limited) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from threading import Lock - -from th2_common.schema.grpc.configuration.grpc_configuration import GrpcRawRobinStrategy -from th2_common.schema.strategy.route.routing_strategy import RoutingStrategy - - -class Robin(RoutingStrategy): - - def __init__(self, configuration) -> None: - self.endpoints = GrpcRawRobinStrategy(**configuration).endpoints - self.index = 0 - self.lock = Lock() - - def get_endpoint(self, request): - with self.lock: - result = self.endpoints[self.index % len(self.endpoints)] - self.index = self.index + 1 - return result diff --git a/th2_common/schema/util/util.py b/th2_common/schema/util/util.py index b6d7249..9dda237 100644 --- a/th2_common/schema/util/util.py +++ b/th2_common/schema/util/util.py @@ -1,4 +1,4 @@ -# Copyright 2021-2021 Exactpro (Exactpro Systems Limited) +# Copyright 2021-2022 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,28 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple - -from google.protobuf.duration_pb2 import Duration -from google.protobuf.pyext._message import RepeatedCompositeContainer -from th2_grpc_common.common_pb2 import MessageID, MessageGroupBatch, AnyMessage, EventBatch, Direction, Value, \ - ListValue, Message, MessageMetadata, ConnectionID, MetadataFilter, RootMessageFilter, MessageFilter, \ - ValueFilter, ListValueFilter, SimpleList, RootComparisonSettings, MessageGroup +from typing import List, Optional, Tuple +from th2_common.schema.message.configuration.message_configuration import MessageRouterConfiguration, \ + MqRouterFilterConfiguration import th2_common.schema.metrics.common_metrics as common_metrics -from th2_common.schema.message.configuration.message_configuration import MessageRouterConfiguration +from th2_grpc_common.common_pb2 import AnyMessage, Direction, EventBatch, MessageGroup, MessageGroupBatch, MessageID -def get_filters(configuration: MessageRouterConfiguration, aliases: List[str]): +def get_filters(configuration: MessageRouterConfiguration, + aliases: List[str]) -> List[List[MqRouterFilterConfiguration]]: return [configuration.queues[alias].filters for alias in aliases] -def get_sequence(group: MessageGroup): - if group.messages[0].HasField('raw_message'): +def get_sequence(group: MessageGroup) -> Optional[int]: + if group.messages[0].HasField('byte_message'): + return group.messages[0].byte_message.metadata.id.sequence + elif group.messages[0].HasField('raw_message'): return group.messages[0].raw_message.metadata.id.sequence elif group.messages[0].HasField('message'): return group.messages[0].message.metadata.id.sequence + return None + def get_session_alias_and_direction(message_id: MessageID) -> Tuple[str, str]: return message_id.connection_id.session_alias, Direction.Name(message_id.direction) @@ -44,6 +45,8 @@ def get_session_alias_and_direction_group(any_message: AnyMessage) -> Tuple[str, return get_session_alias_and_direction(any_message.message.metadata.id) elif any_message.HasField('raw_message'): return get_session_alias_and_direction(any_message.raw_message.metadata.id) + elif any_message.HasField('byte_message'): + return get_session_alias_and_direction(any_message.byte_message.metadata.id) else: return common_metrics.UNKNOWN_LABELS @@ -51,7 +54,7 @@ def get_session_alias_and_direction_group(any_message: AnyMessage) -> Tuple[str, def get_debug_string(class_name: str, ids: List[MessageID]) -> str: session_alias, direction = get_session_alias_and_direction(ids[0]) sequences = ', '.join([str(i.sequence) for i in ids]) - return f'{class_name}: {session_alias = }, {direction = }, {sequences = }'.strip() + return f'{class_name}: session_alias = {session_alias}, direction = {direction}, sequences = {sequences}'.strip() def get_debug_string_event(event_batch: EventBatch) -> str: @@ -62,98 +65,17 @@ def get_debug_string_group(group_batch: MessageGroupBatch) -> str: messages = [message for group in group_batch.groups for message in group.messages] session_alias, direction = get_session_alias_and_direction_group(messages[0]) sequences = [] + for message in messages: if message.HasField('message'): sequences.append(str(message.message.metadata.id.sequence)) elif message.HasField('raw_message'): sequences.append(str(message.raw_message.metadata.id.sequence)) - sequences = ''.join(sequences) + elif message.HasField('byte_message'): + sequences.append(str(message.byte_message.metadata.id.sequence)) + sequences_string = ''.join(sequences) return f'MessageGroupBatch: ' \ - f'{session_alias = }, ' \ - f'{direction = }, ' \ - f'{sequences = }'.strip() - - -def convert_message_value(value): - if isinstance(value, Value): - return value - elif isinstance(value, (str, int, float)): - return Value(simple_value=str(value)) - elif isinstance(value, list): - return Value(list_value=ListValue(values=[convert_message_value(x) for x in value])) - elif isinstance(value, dict): - return Value(message_value=Message(fields={key: convert_message_value(value[key]) for key in value})) - - -def create_message(fields: dict, session_alias=None, message_type=None): - return Message(metadata=MessageMetadata(id=MessageID(connection_id=ConnectionID(session_alias=session_alias)), - message_type=message_type), - fields={field: convert_message_value(fields[field]) for field in fields}) - - -def convert_filter_value(value, message_type=None, direction=None, values=False, metadata=False): - if isinstance(value, ValueFilter): - return value - elif isinstance(value, (str, int, float)) and values is True: - return ValueFilter(simple_filter=str(value)) - elif isinstance(value, (str, int, float)) and metadata is True: - return MetadataFilter.SimpleFilter(value=str(value)) - elif isinstance(value, list) and values is True: - return ValueFilter(list_filter=ListValueFilter( - values=[convert_filter_value(x, values=values, metadata=metadata) for x in value])) - elif isinstance(value, list) and metadata is True: - return MetadataFilter.SimpleFilter(simple_list=SimpleList(simple_values=value)) - elif isinstance(value, dict): - return ValueFilter( - message_filter=MessageFilter(messageType=message_type, - fields={key: convert_filter_value(value[key], - values=values, - metadata=metadata) - for key in value}, - direction=direction)) - - -def create_root_message_filter(message_type=None, - message_filter=None, - metadata_filter=None, - ignore_fields: List[str] = None, - check_repeating_group_order: bool = None, - time_precision: Duration = None, - decimal_precision: str = None): - if message_filter is None: - message_filter = {} - if metadata_filter is None: - metadata_filter = {} - return RootMessageFilter(messageType=message_type, - message_filter=MessageFilter(fields={ - field: convert_filter_value(message_filter[field], values=True) - for field in message_filter}), - metadata_filter=MetadataFilter(property_filters={ - value: convert_filter_value(metadata_filter[value], metadata=True) - for value in metadata_filter}), - comparison_settings=RootComparisonSettings( - ignore_fields=ignore_fields, - check_repeating_group_order=check_repeating_group_order, - time_precision=time_precision, - decimal_precision=decimal_precision)) - - -def convert_value_into_typed_field(field_value, typed_field_value): - if field_value.WhichOneof('kind') == 'simple_value': - return type(typed_field_value)(field_value.simple_value) - elif field_value.WhichOneof('kind') == 'list_value': - return [convert_value_into_typed_field(list_item, typed_field_value.add()) - for list_item in field_value.list_value.values] - elif field_value.WhichOneof('kind') == 'message_value': - fields_typed = {field: convert_value_into_typed_field(field_value.message_value.fields[field], - getattr(typed_field_value, field)) - for field in field_value.message_value.fields} - return type(typed_field_value)(**fields_typed) - - -def create_typed_message_from_message(message, message_type): - response_fields = [field.name for field in message_type().DESCRIPTOR.fields] - fields_typed = {field: convert_value_into_typed_field(message.fields[field], getattr(message_type(), field)) - for field in response_fields} - return message_type(**fields_typed) + f'session_alias = {session_alias}, ' \ + f'direction = {direction}, ' \ + f'sequences_string = {sequences_string}'.strip()